Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-27 01:54:47 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: one big snapshot

Browse Source
This commit is contained in:
nym21
2025-08-02 16:59:22 +02:00
parent aa8b47a3dd
commit f7aa9424db
252 changed files with 6283 additions and 5264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
crates/brk_indexer/src/indexes.rs
View File

@@ -1,13 +1,15 @@
use bitcoincore_rpc::Client;
use brk_core::{
use brk_error::{Error, Result};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_structs::{
BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex,
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
Result, TxIndex, TypeIndex, UnknownOutputIndex,
TxIndex, TypeIndex, UnknownOutputIndex,
};
use brk_vecs::{
AnyIterableVec, AnyStoredIterableVec, GenericStoredVec, StoredIndex, StoredRaw, VecIterator,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vecs::{AnyIterableVec, AnyStampedVec, AnyVec, StampedVec, StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::{Stores, Vecs};
@@ -46,6 +48,7 @@ impl Indexes {
OutputType::P2WPKH => *self.p2wpkhaddressindex,
OutputType::P2WSH => *self.p2wshaddressindex,
OutputType::Unknown => *self.unknownoutputindex,
_ => unreachable!(),
}
}
@@ -87,8 +90,8 @@ impl Indexes {
}
impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
type Error = color_eyre::Report;
fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result<Self> {
type Error = Error;
fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> Result<Self> {
// Height at which we want to start: min last saved + 1 or 0
let vecs_starting_height = vecs.starting_height();
let stores_starting_height = stores.starting_height();
@@ -123,100 +126,100 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
&vecs.emptyoutputindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
height,
p2msoutputindex: starting_index(
&vecs.height_to_first_p2msoutputindex,
&vecs.p2msoutputindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
opreturnindex: starting_index(
&vecs.height_to_first_opreturnindex,
&vecs.opreturnindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2pk33addressindex: starting_index(
&vecs.height_to_first_p2pk33addressindex,
&vecs.p2pk33addressindex_to_p2pk33bytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2pk65addressindex: starting_index(
&vecs.height_to_first_p2pk65addressindex,
&vecs.p2pk65addressindex_to_p2pk65bytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2pkhaddressindex: starting_index(
&vecs.height_to_first_p2pkhaddressindex,
&vecs.p2pkhaddressindex_to_p2pkhbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2shaddressindex: starting_index(
&vecs.height_to_first_p2shaddressindex,
&vecs.p2shaddressindex_to_p2shbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2traddressindex: starting_index(
&vecs.height_to_first_p2traddressindex,
&vecs.p2traddressindex_to_p2trbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2wpkhaddressindex: starting_index(
&vecs.height_to_first_p2wpkhaddressindex,
&vecs.p2wpkhaddressindex_to_p2wpkhbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2wshaddressindex: starting_index(
&vecs.height_to_first_p2wshaddressindex,
&vecs.p2wshaddressindex_to_p2wshbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2aaddressindex: starting_index(
&vecs.height_to_first_p2aaddressindex,
&vecs.p2aaddressindex_to_p2abytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
txindex: starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)
.context("")?,
.ok_or(Error::Str(""))?,
inputindex: starting_index(
&vecs.height_to_first_inputindex,
&vecs.inputindex_to_outputindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
outputindex: starting_index(
&vecs.height_to_first_outputindex,
&vecs.outputindex_to_value,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
unknownoutputindex: starting_index(
&vecs.height_to_first_unknownoutputindex,
&vecs.unknownoutputindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
})
}
}
pub fn starting_index<I, T>(
height_to_index: &StampedVec<Height, I>,
index_to_else: &StampedVec<I, T>,
height_to_index: &impl AnyStoredIterableVec<Height, I>,
index_to_else: &impl AnyIterableVec<I, T>,
starting_height: Height,
) -> Option<I>
where
I: StoredType + StoredIndex + From<usize>,
T: StoredType,
I: StoredRaw + StoredIndex + From<usize>,
T: StoredRaw,
{
let h = Height::from(u64::from(height_to_index.stamp()));
if h.is_zero() {

88
crates/brk_indexer/src/lib.rs
View File

@@ -6,16 +6,16 @@
use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant};
use bitcoin::{Transaction, TxIn, TxOut};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Result, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit,
};
use brk_exit::Exit;
use brk_error::{Error, Result};
use brk_parser::Parser;
use brk_store::AnyStore;
use brk_vecs::{AnyVec, File, PAGE_SIZE, Reader, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use brk_structs::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Sats, StoredBool, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout,
};
use brk_vecs::{AnyVec, Exit, File, GenericStoredVec, PAGE_SIZE, Reader, VecIterator};
use log::{error, info};
use rayon::prelude::*;
mod indexes;
@@ -38,9 +38,7 @@ pub struct Indexer {
}
impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result<Self> {
setrlimit()?;
pub fn forced_import(outputs_dir: &Path) -> Result<Self> {
let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?);
let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?;
@@ -63,7 +61,7 @@ impl Indexer {
rpc: &'static bitcoincore_rpc::Client,
exit: &Exit,
check_collisions: bool,
) -> color_eyre::Result<Indexes> {
) -> Result<Indexes> {
let file = self.file.clone();
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc))
@@ -177,7 +175,7 @@ impl Indexer {
);
parser.parse(start, end).iter().try_for_each(
|(height, block, blockhash)| -> color_eyre::Result<()> {
|(height, block, blockhash)| -> Result<()> {
info!("Indexing block {height}...");
idxs.height = height;
@@ -204,7 +202,7 @@ impl Indexer {
.is_some_and(|prev_height| *prev_height != height)
{
error!("BlockHash: {blockhash}");
return Err(eyre!("Collision, expect prefix to need be set yet"));
return Err(Error::Str("Collision, expect prefix to need be set yet"));
}
idxs.push_if_needed(vecs)?;
@@ -227,7 +225,7 @@ impl Indexer {
outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle,
) = thread::scope(|scope| {
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle =
scope.spawn(|| -> color_eyre::Result<_> {
scope.spawn(|| -> Result<_> {
block
.txdata
.iter()
@@ -247,7 +245,7 @@ impl Indexer {
Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt)))
})
.collect::<color_eyre::Result<BTreeMap<_, _>>>()
.collect::<Result<BTreeMap<_, _>>>()
});
let input_source_vec_handle = scope.spawn(|| {
@@ -266,7 +264,7 @@ impl Indexer {
inputs
.into_par_iter()
.enumerate()
.map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> color_eyre::Result<(InputIndex, InputSource)> {
.map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> Result<(InputIndex, InputSource)> {
let txindex = idxs.txindex + block_txindex;
let inputindex = idxs.inputindex + InputIndex::from(block_inputindex);
@@ -294,7 +292,7 @@ impl Indexer {
let vout = Vout::from(outpoint.vout);
let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_mmap)?
.context("Expect outputindex to not be none")
.ok_or(Error::Str("Expect outputindex to not be none"))
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
})?.into_owned()
@@ -306,7 +304,7 @@ impl Indexer {
outputindex,
))))
})
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
.try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
@@ -337,14 +335,14 @@ impl Indexer {
.enumerate()
.map(
#[allow(clippy::type_complexity)]
|(block_outputindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<(
|(block_outputindex, (block_txindex, vout, txout, tx))| -> Result<(
OutputIndex,
(
&TxOut,
TxIndex,
Vout,
OutputType,
brk_core::Result<AddressBytes>,
Result<AddressBytes>,
Option<TypeIndex>,
&Transaction,
),
@@ -409,12 +407,12 @@ impl Indexer {
.p2aaddressindex_to_p2abytes
.get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
_ => {
unreachable!()
}
};
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
prev_addressbytes_opt.as_ref().ok_or(Error::Str("Expect to have addressbytes"))?;
if stores.addressbyteshash_to_typeindex.needs(height)
&& prev_addressbytes != addressbytes
@@ -452,7 +450,7 @@ impl Indexer {
))
},
)
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
.try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
@@ -476,20 +474,19 @@ impl Indexer {
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex =
txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle
.ok()
.context(
"Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join",
.map_err(|_|
Error::Str("Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join")
)??;
let input_source_vec = input_source_vec_handle
.ok()
.context("Export input_source_vec_handle to join")??;
.map_err(|_|
Error::Str("Export input_source_vec_handle to join")
)??;
let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt =
outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle
.ok()
.context(
"Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join",
.map_err(|_|
Error::Str("Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join")
)?;
let outputs_len = outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt.len();
@@ -510,7 +507,7 @@ impl Indexer {
outputindex,
(txout, txindex, vout, outputtype, addressbytes_res, typeindex_opt, _tx),
)|
-> color_eyre::Result<()> {
-> Result<()> {
let sats = Sats::from(txout.value);
if vout.is_zero() {
@@ -580,6 +577,7 @@ impl Indexer {
vecs.unknownoutputindex_to_txindex.push_if_needed(idxs.unknownoutputindex, txindex)?;
idxs.unknownoutputindex.copy_then_increment()
},
_ => unreachable!()
};
if let Ok(addressbytes) = addressbytes_res {
@@ -618,7 +616,7 @@ impl Indexer {
.into_iter()
.map(
#[allow(clippy::type_complexity)]
|(inputindex, input_source)| -> color_eyre::Result<(
|(inputindex, input_source)| -> Result<(
InputIndex, Vin, TxIndex, OutputIndex
)> {
match input_source {
@@ -634,7 +632,7 @@ impl Indexer {
let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.get(&TxidPrefix::from(&txid))
.context("txid should be in same block").inspect_err(|_| {
.ok_or(Error::Str("txid should be in same block")).inspect_err(|_| {
dbg!(&txid_prefix_to_txid_and_block_txindex_and_prev_txindex);
// panic!();
})?
@@ -643,7 +641,7 @@ impl Indexer {
let prev_outputindex = new_txindexvout_to_outputindex
.remove(&(prev_txindex, vout))
.context("should have found addressindex from same block")
.ok_or(Error::Str("should have found addressindex from same block"))
.inspect_err(|_| {
dbg!(&new_txindexvout_to_outputindex, txin, prev_txindex, vout, txid);
})?;
@@ -653,7 +651,7 @@ impl Indexer {
}
},
)
.try_for_each(|res| -> color_eyre::Result<()> {
.try_for_each(|res| -> Result<()> {
let (inputindex, vin, txindex, outputindex) = res?;
if vin.is_zero() {
@@ -675,7 +673,7 @@ impl Indexer {
txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.into_iter()
.try_for_each(
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> Result<()> {
let txindex = idxs.txindex + index;
txindex_to_tx_and_txid.insert(txindex, (tx, txid));
@@ -700,7 +698,7 @@ impl Indexer {
// Ok if `get` is not par as should happen only twice
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.context("To have txid for txindex")
.ok_or(Error::Str("To have txid for txindex"))
.inspect_err(|_| {
dbg!(txindex, len);
})?;
@@ -712,11 +710,13 @@ impl Indexer {
let only_known_dup_txids = [
bitcoin::Txid::from_str(
"d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599",
)?
)
.unwrap()
.into(),
bitcoin::Txid::from_str(
"e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468",
)?
)
.unwrap()
.into(),
];
@@ -724,7 +724,7 @@ impl Indexer {
if !is_dup {
dbg!(height, txindex, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
return Err(Error::Str("Expect none"));
}
}
}
@@ -737,13 +737,13 @@ impl Indexer {
txindex_to_tx_and_txid
.into_iter()
.try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
.try_for_each(|(txindex, (tx, txid))| -> Result<()> {
vecs.txindex_to_txversion.push_if_needed(txindex, tx.version.into())?;
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
vecs.txindex_to_rawlocktime.push_if_needed(txindex, tx.lock_time.into())?;
vecs.txindex_to_base_size.push_if_needed(txindex, tx.base_size().into())?;
vecs.txindex_to_total_size.push_if_needed(txindex, tx.total_size().into())?;
vecs.txindex_to_is_explicitly_rbf.push_if_needed(txindex, tx.is_explicitly_rbf())?;
vecs.txindex_to_is_explicitly_rbf.push_if_needed(txindex, StoredBool::from(tx.is_explicitly_rbf()))?;
Ok(())
})?;

15
crates/brk_indexer/src/stores.rs
View File

@@ -1,11 +1,12 @@
use std::{borrow::Cow, fs, path::Path, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex,
OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
};
use brk_error::Result;
use brk_store::{AnyStore, Store};
use brk_vecs::{AnyIterableVec, VecIterator};
use brk_structs::{
AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex,
OutputType, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
};
use brk_vecs::VecIterator;
use fjall::{PersistMode, TransactionalKeyspace};
use rayon::prelude::*;
@@ -27,7 +28,7 @@ pub struct Stores {
const VERSION: Version = Version::ZERO;
impl Stores {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
fs::create_dir_all(path)?;
let keyspace = match brk_store::open_keyspace(path) {
@@ -217,7 +218,7 @@ impl Stores {
&mut self,
vecs: &mut Vecs,
starting_indexes: &Indexes,
) -> color_eyre::Result<()> {
) -> Result<()> {
if self.addressbyteshash_to_typeindex.is_empty()?
&& self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?

347
crates/brk_indexer/src/vecs.rs
View File

@@ -1,14 +1,17 @@
use std::sync::Arc;
use brk_core::{
use brk_error::Result;
use brk_structs::{
AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, P2AAddressIndex, P2ABytes, P2MSOutputIndex, P2PK33AddressIndex, P2PK33Bytes,
P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex, P2PKHBytes, P2SHAddressIndex, P2SHBytes,
P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes,
RawLockTime, Result, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion,
RawLockTime, Sats, StoredBool, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion,
Txid, TypeIndex, UnknownOutputIndex, Version, Weight,
};
use brk_vecs::{AnyCollectableVec, AnyStampedVec, File, Format, StampedVec};
use brk_vecs::{
AnyCollectableVec, AnyStoredVec, CompressedVec, File, GenericStoredVec, RawVec, Stamp,
};
use rayon::prelude::*;
use crate::Indexes;
@@ -17,320 +20,276 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub emptyoutputindex_to_txindex: StampedVec<EmptyOutputIndex, TxIndex>,
pub height_to_blockhash: StampedVec<Height, BlockHash>,
pub height_to_difficulty: StampedVec<Height, StoredF64>,
pub height_to_first_emptyoutputindex: StampedVec<Height, EmptyOutputIndex>,
pub height_to_first_inputindex: StampedVec<Height, InputIndex>,
pub height_to_first_opreturnindex: StampedVec<Height, OpReturnIndex>,
pub height_to_first_outputindex: StampedVec<Height, OutputIndex>,
pub height_to_first_p2aaddressindex: StampedVec<Height, P2AAddressIndex>,
pub height_to_first_p2msoutputindex: StampedVec<Height, P2MSOutputIndex>,
pub height_to_first_p2pk33addressindex: StampedVec<Height, P2PK33AddressIndex>,
pub height_to_first_p2pk65addressindex: StampedVec<Height, P2PK65AddressIndex>,
pub height_to_first_p2pkhaddressindex: StampedVec<Height, P2PKHAddressIndex>,
pub height_to_first_p2shaddressindex: StampedVec<Height, P2SHAddressIndex>,
pub height_to_first_p2traddressindex: StampedVec<Height, P2TRAddressIndex>,
pub height_to_first_p2wpkhaddressindex: StampedVec<Height, P2WPKHAddressIndex>,
pub height_to_first_p2wshaddressindex: StampedVec<Height, P2WSHAddressIndex>,
pub height_to_first_txindex: StampedVec<Height, TxIndex>,
pub height_to_first_unknownoutputindex: StampedVec<Height, UnknownOutputIndex>,
pub emptyoutputindex_to_txindex: CompressedVec<EmptyOutputIndex, TxIndex>,
pub height_to_blockhash: RawVec<Height, BlockHash>,
pub height_to_difficulty: CompressedVec<Height, StoredF64>,
pub height_to_first_emptyoutputindex: CompressedVec<Height, EmptyOutputIndex>,
pub height_to_first_inputindex: CompressedVec<Height, InputIndex>,
pub height_to_first_opreturnindex: CompressedVec<Height, OpReturnIndex>,
pub height_to_first_outputindex: CompressedVec<Height, OutputIndex>,
pub height_to_first_p2aaddressindex: CompressedVec<Height, P2AAddressIndex>,
pub height_to_first_p2msoutputindex: CompressedVec<Height, P2MSOutputIndex>,
pub height_to_first_p2pk33addressindex: CompressedVec<Height, P2PK33AddressIndex>,
pub height_to_first_p2pk65addressindex: CompressedVec<Height, P2PK65AddressIndex>,
pub height_to_first_p2pkhaddressindex: CompressedVec<Height, P2PKHAddressIndex>,
pub height_to_first_p2shaddressindex: CompressedVec<Height, P2SHAddressIndex>,
pub height_to_first_p2traddressindex: CompressedVec<Height, P2TRAddressIndex>,
pub height_to_first_p2wpkhaddressindex: CompressedVec<Height, P2WPKHAddressIndex>,
pub height_to_first_p2wshaddressindex: CompressedVec<Height, P2WSHAddressIndex>,
pub height_to_first_txindex: CompressedVec<Height, TxIndex>,
pub height_to_first_unknownoutputindex: CompressedVec<Height, UnknownOutputIndex>,
/// Doesn't guarantee continuity due to possible reorgs
pub height_to_timestamp: StampedVec<Height, Timestamp>,
pub height_to_total_size: StampedVec<Height, StoredUsize>,
pub height_to_weight: StampedVec<Height, Weight>,
pub height_to_timestamp: CompressedVec<Height, Timestamp>,
pub height_to_total_size: CompressedVec<Height, StoredU64>,
pub height_to_weight: CompressedVec<Height, Weight>,
/// If outputindex == Outputindex::MAX then it's coinbase
pub inputindex_to_outputindex: StampedVec<InputIndex, OutputIndex>,
pub opreturnindex_to_txindex: StampedVec<OpReturnIndex, TxIndex>,
pub outputindex_to_outputtype: StampedVec<OutputIndex, OutputType>,
pub outputindex_to_typeindex: StampedVec<OutputIndex, TypeIndex>,
pub outputindex_to_value: StampedVec<OutputIndex, Sats>,
pub p2aaddressindex_to_p2abytes: StampedVec<P2AAddressIndex, P2ABytes>,
pub p2msoutputindex_to_txindex: StampedVec<P2MSOutputIndex, TxIndex>,
pub p2pk33addressindex_to_p2pk33bytes: StampedVec<P2PK33AddressIndex, P2PK33Bytes>,
pub p2pk65addressindex_to_p2pk65bytes: StampedVec<P2PK65AddressIndex, P2PK65Bytes>,
pub p2pkhaddressindex_to_p2pkhbytes: StampedVec<P2PKHAddressIndex, P2PKHBytes>,
pub p2shaddressindex_to_p2shbytes: StampedVec<P2SHAddressIndex, P2SHBytes>,
pub p2traddressindex_to_p2trbytes: StampedVec<P2TRAddressIndex, P2TRBytes>,
pub p2wpkhaddressindex_to_p2wpkhbytes: StampedVec<P2WPKHAddressIndex, P2WPKHBytes>,
pub p2wshaddressindex_to_p2wshbytes: StampedVec<P2WSHAddressIndex, P2WSHBytes>,
pub txindex_to_base_size: StampedVec<TxIndex, StoredU32>,
pub txindex_to_first_inputindex: StampedVec<TxIndex, InputIndex>,
pub txindex_to_first_outputindex: StampedVec<TxIndex, OutputIndex>,
pub txindex_to_is_explicitly_rbf: StampedVec<TxIndex, bool>,
pub txindex_to_rawlocktime: StampedVec<TxIndex, RawLockTime>,
pub txindex_to_total_size: StampedVec<TxIndex, StoredU32>,
pub txindex_to_txid: StampedVec<TxIndex, Txid>,
pub txindex_to_txversion: StampedVec<TxIndex, TxVersion>,
pub unknownoutputindex_to_txindex: StampedVec<UnknownOutputIndex, TxIndex>,
pub inputindex_to_outputindex: RawVec<InputIndex, OutputIndex>,
pub opreturnindex_to_txindex: CompressedVec<OpReturnIndex, TxIndex>,
pub outputindex_to_outputtype: RawVec<OutputIndex, OutputType>,
pub outputindex_to_typeindex: RawVec<OutputIndex, TypeIndex>,
pub outputindex_to_value: RawVec<OutputIndex, Sats>,
pub p2aaddressindex_to_p2abytes: RawVec<P2AAddressIndex, P2ABytes>,
pub p2msoutputindex_to_txindex: CompressedVec<P2MSOutputIndex, TxIndex>,
pub p2pk33addressindex_to_p2pk33bytes: RawVec<P2PK33AddressIndex, P2PK33Bytes>,
pub p2pk65addressindex_to_p2pk65bytes: RawVec<P2PK65AddressIndex, P2PK65Bytes>,
pub p2pkhaddressindex_to_p2pkhbytes: RawVec<P2PKHAddressIndex, P2PKHBytes>,
pub p2shaddressindex_to_p2shbytes: RawVec<P2SHAddressIndex, P2SHBytes>,
pub p2traddressindex_to_p2trbytes: RawVec<P2TRAddressIndex, P2TRBytes>,
pub p2wpkhaddressindex_to_p2wpkhbytes: RawVec<P2WPKHAddressIndex, P2WPKHBytes>,
pub p2wshaddressindex_to_p2wshbytes: RawVec<P2WSHAddressIndex, P2WSHBytes>,
pub txindex_to_base_size: CompressedVec<TxIndex, StoredU32>,
pub txindex_to_first_inputindex: CompressedVec<TxIndex, InputIndex>,
pub txindex_to_first_outputindex: CompressedVec<TxIndex, OutputIndex>,
pub txindex_to_is_explicitly_rbf: CompressedVec<TxIndex, StoredBool>,
pub txindex_to_rawlocktime: CompressedVec<TxIndex, RawLockTime>,
pub txindex_to_total_size: CompressedVec<TxIndex, StoredU32>,
pub txindex_to_txid: RawVec<TxIndex, Txid>,
pub txindex_to_txversion: CompressedVec<TxIndex, TxVersion>,
pub unknownoutputindex_to_txindex: CompressedVec<UnknownOutputIndex, TxIndex>,
}
impl Vecs {
pub fn forced_import(file: &Arc<File>, version: Version) -> color_eyre::Result<Self> {
pub fn forced_import(file: &Arc<File>, version: Version) -> Result<Self> {
Ok(Self {
emptyoutputindex_to_txindex: StampedVec::forced_import(
emptyoutputindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_blockhash: StampedVec::forced_import(
height_to_blockhash: RawVec::forced_import(
file,
"blockhash",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_difficulty: StampedVec::forced_import(
height_to_difficulty: CompressedVec::forced_import(
file,
"difficulty",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_emptyoutputindex: StampedVec::forced_import(
height_to_first_emptyoutputindex: CompressedVec::forced_import(
file,
"first_emptyoutputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_inputindex: StampedVec::forced_import(
height_to_first_inputindex: CompressedVec::forced_import(
file,
"first_inputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_opreturnindex: StampedVec::forced_import(
height_to_first_opreturnindex: CompressedVec::forced_import(
file,
"first_opreturnindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_outputindex: StampedVec::forced_import(
height_to_first_outputindex: CompressedVec::forced_import(
file,
"first_outputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2aaddressindex: StampedVec::forced_import(
height_to_first_p2aaddressindex: CompressedVec::forced_import(
file,
"first_p2aaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2msoutputindex: StampedVec::forced_import(
height_to_first_p2msoutputindex: CompressedVec::forced_import(
file,
"first_p2msoutputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2pk33addressindex: StampedVec::forced_import(
height_to_first_p2pk33addressindex: CompressedVec::forced_import(
file,
"first_p2pk33addressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2pk65addressindex: StampedVec::forced_import(
height_to_first_p2pk65addressindex: CompressedVec::forced_import(
file,
"first_p2pk65addressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2pkhaddressindex: StampedVec::forced_import(
height_to_first_p2pkhaddressindex: CompressedVec::forced_import(
file,
"first_p2pkhaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2shaddressindex: StampedVec::forced_import(
height_to_first_p2shaddressindex: CompressedVec::forced_import(
file,
"first_p2shaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2traddressindex: StampedVec::forced_import(
height_to_first_p2traddressindex: CompressedVec::forced_import(
file,
"first_p2traddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2wpkhaddressindex: StampedVec::forced_import(
height_to_first_p2wpkhaddressindex: CompressedVec::forced_import(
file,
"first_p2wpkhaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2wshaddressindex: StampedVec::forced_import(
height_to_first_p2wshaddressindex: CompressedVec::forced_import(
file,
"first_p2wshaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_txindex: StampedVec::forced_import(
height_to_first_txindex: CompressedVec::forced_import(
file,
"first_txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_unknownoutputindex: StampedVec::forced_import(
height_to_first_unknownoutputindex: CompressedVec::forced_import(
file,
"first_unknownoutputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_timestamp: StampedVec::forced_import(
height_to_timestamp: CompressedVec::forced_import(
file,
"timestamp",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_total_size: StampedVec::forced_import(
height_to_total_size: CompressedVec::forced_import(
file,
"total_size",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_weight: StampedVec::forced_import(
height_to_weight: CompressedVec::forced_import(
file,
"weight",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
inputindex_to_outputindex: StampedVec::forced_import(
inputindex_to_outputindex: RawVec::forced_import(
file,
"outputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
opreturnindex_to_txindex: StampedVec::forced_import(
opreturnindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
outputindex_to_outputtype: StampedVec::forced_import(
outputindex_to_outputtype: RawVec::forced_import(
file,
"outputtype",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
outputindex_to_typeindex: StampedVec::forced_import(
outputindex_to_typeindex: RawVec::forced_import(
file,
"typeindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
outputindex_to_value: StampedVec::forced_import(
outputindex_to_value: RawVec::forced_import(
file,
"value",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2aaddressindex_to_p2abytes: StampedVec::forced_import(
p2aaddressindex_to_p2abytes: RawVec::forced_import(
file,
"p2abytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2msoutputindex_to_txindex: StampedVec::forced_import(
p2msoutputindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk33addressindex_to_p2pk33bytes: StampedVec::forced_import(
p2pk33addressindex_to_p2pk33bytes: RawVec::forced_import(
file,
"p2pk33bytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk65addressindex_to_p2pk65bytes: StampedVec::forced_import(
p2pk65addressindex_to_p2pk65bytes: RawVec::forced_import(
file,
"p2pk65bytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pkhaddressindex_to_p2pkhbytes: StampedVec::forced_import(
p2pkhaddressindex_to_p2pkhbytes: RawVec::forced_import(
file,
"p2pkhbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2shaddressindex_to_p2shbytes: StampedVec::forced_import(
p2shaddressindex_to_p2shbytes: RawVec::forced_import(
file,
"p2shbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2traddressindex_to_p2trbytes: StampedVec::forced_import(
p2traddressindex_to_p2trbytes: RawVec::forced_import(
file,
"p2trbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wpkhaddressindex_to_p2wpkhbytes: StampedVec::forced_import(
p2wpkhaddressindex_to_p2wpkhbytes: RawVec::forced_import(
file,
"p2wpkhbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wshaddressindex_to_p2wshbytes: StampedVec::forced_import(
p2wshaddressindex_to_p2wshbytes: RawVec::forced_import(
file,
"p2wshbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_base_size: StampedVec::forced_import(
txindex_to_base_size: CompressedVec::forced_import(
file,
"base_size",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_first_inputindex: StampedVec::forced_import(
txindex_to_first_inputindex: CompressedVec::forced_import(
file,
"first_inputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_first_outputindex: StampedVec::forced_import(
txindex_to_first_outputindex: CompressedVec::forced_import(
file,
"first_outputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_is_explicitly_rbf: StampedVec::forced_import(
txindex_to_is_explicitly_rbf: CompressedVec::forced_import(
file,
"is_explicitly_rbf",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_rawlocktime: StampedVec::forced_import(
txindex_to_rawlocktime: CompressedVec::forced_import(
file,
"rawlocktime",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_total_size: StampedVec::forced_import(
txindex_to_total_size: CompressedVec::forced_import(
file,
"total_size",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_txid: StampedVec::forced_import(
txindex_to_txid: RawVec::forced_import(
file,
"txid",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_txversion: StampedVec::forced_import(
txindex_to_txversion: CompressedVec::forced_import(
file,
"txversion",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
unknownoutputindex_to_txindex: StampedVec::forced_import(
unknownoutputindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
})
}
@@ -360,89 +319,93 @@ impl Vecs {
let stamp = u64::from(saved_height).into();
self.emptyoutputindex_to_txindex
.truncate_if_needed(emptyoutputindex, stamp)?;
self.height_to_blockhash.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(emptyoutputindex, stamp)?;
self.height_to_blockhash
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_difficulty
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_emptyoutputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_inputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_opreturnindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_outputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2aaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2msoutputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2pk33addressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2pk65addressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2pkhaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2shaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2traddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2wpkhaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2wshaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_txindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_unknownoutputindex
.truncate_if_needed(height, stamp)?;
self.height_to_timestamp.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_timestamp
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_total_size
.truncate_if_needed(height, stamp)?;
self.height_to_weight.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_weight
.truncate_if_needed_with_stamp(height, stamp)?;
self.inputindex_to_outputindex
.truncate_if_needed(inputindex, stamp)?;
.truncate_if_needed_with_stamp(inputindex, stamp)?;
self.opreturnindex_to_txindex
.truncate_if_needed(opreturnindex, stamp)?;
.truncate_if_needed_with_stamp(opreturnindex, stamp)?;
self.outputindex_to_outputtype
.truncate_if_needed(outputindex, stamp)?;
.truncate_if_needed_with_stamp(outputindex, stamp)?;
self.outputindex_to_typeindex
.truncate_if_needed(outputindex, stamp)?;
.truncate_if_needed_with_stamp(outputindex, stamp)?;
self.outputindex_to_value
.truncate_if_needed(outputindex, stamp)?;
.truncate_if_needed_with_stamp(outputindex, stamp)?;
self.p2aaddressindex_to_p2abytes
.truncate_if_needed(p2aaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2aaddressindex, stamp)?;
self.p2msoutputindex_to_txindex
.truncate_if_needed(p2msoutputindex, stamp)?;
.truncate_if_needed_with_stamp(p2msoutputindex, stamp)?;
self.p2pk33addressindex_to_p2pk33bytes
.truncate_if_needed(p2pk33addressindex, stamp)?;
.truncate_if_needed_with_stamp(p2pk33addressindex, stamp)?;
self.p2pk65addressindex_to_p2pk65bytes
.truncate_if_needed(p2pk65addressindex, stamp)?;
.truncate_if_needed_with_stamp(p2pk65addressindex, stamp)?;
self.p2pkhaddressindex_to_p2pkhbytes
.truncate_if_needed(p2pkhaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2pkhaddressindex, stamp)?;
self.p2shaddressindex_to_p2shbytes
.truncate_if_needed(p2shaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2shaddressindex, stamp)?;
self.p2traddressindex_to_p2trbytes
.truncate_if_needed(p2traddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2traddressindex, stamp)?;
self.p2wpkhaddressindex_to_p2wpkhbytes
.truncate_if_needed(p2wpkhaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2wpkhaddressindex, stamp)?;
self.p2wshaddressindex_to_p2wshbytes
.truncate_if_needed(p2wshaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2wshaddressindex, stamp)?;
self.txindex_to_base_size
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_first_inputindex
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_first_outputindex
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_is_explicitly_rbf
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_rawlocktime
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_total_size
.truncate_if_needed(txindex, stamp)?;
self.txindex_to_txid.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_txid
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_txversion
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.unknownoutputindex_to_txindex
.truncate_if_needed(unknownoutputindex, stamp)?;
.truncate_if_needed_with_stamp(unknownoutputindex, stamp)?;
Ok(())
}
@@ -451,35 +414,37 @@ impl Vecs {
match bytes {
AddressBytes::P2PK65(bytes) => self
.p2pk65addressindex_to_p2pk65bytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2PK33(bytes) => self
.p2pk33addressindex_to_p2pk33bytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2PKH(bytes) => self
.p2pkhaddressindex_to_p2pkhbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2SH(bytes) => self
.p2shaddressindex_to_p2shbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2WPKH(bytes) => self
.p2wpkhaddressindex_to_p2wpkhbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2WSH(bytes) => self
.p2wshaddressindex_to_p2wshbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2TR(bytes) => self
.p2traddressindex_to_p2trbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2A(bytes) => self
.p2aaddressindex_to_p2abytes
.push_if_needed(index.into(), bytes),
}
.push_if_needed(index.into(), bytes)?,
};
Ok(())
}
pub fn flush(&mut self, height: Height) -> Result<()> {
self.mut_vecs()
.into_par_iter()
.try_for_each(|vec| vec.flush(u64::from(height).into()))
.try_for_each(|vec| vec.stamped_flush(Stamp::from(u64::from(height))))?;
Ok(())
}
pub fn starting_height(&mut self) -> Height {
@@ -542,7 +507,7 @@ impl Vecs {
]
}
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStampedVec> {
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStoredVec> {
vec![
&mut self.emptyoutputindex_to_txindex,
&mut self.height_to_blockhash,