global: snapshot

This commit is contained in:
nym21
2025-03-10 11:42:15 +01:00
parent f9f7172702
commit 9428beeae5
44 changed files with 1348 additions and 581 deletions

View File

@@ -1,5 +1,6 @@
use std::{path::Path, thread::sleep, time::Duration};
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::{Indexer, rpc::RpcApi};
use brk_parser::{
@@ -13,7 +14,8 @@ fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
let bitcoin_dir = Path::new("../../../bitcoin");
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),

View File

@@ -1,7 +1,8 @@
use bitcoincore_rpc::Client;
use brk_core::{
Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex,
P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
Addressindex, BlockHash, CheckedSub, Emptyindex, Height, Multisigindex, Opreturnindex,
P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex,
Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use color_eyre::eyre::ContextCompat;
@@ -32,8 +33,10 @@ pub struct Indexes {
impl Indexes {
pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> {
let height = self.height;
vecs.height_to_first_txindex.push_if_needed(height, self.txindex)?;
vecs.height_to_first_txinindex.push_if_needed(height, self.txinindex)?;
vecs.height_to_first_txindex
.push_if_needed(height, self.txindex)?;
vecs.height_to_first_txinindex
.push_if_needed(height, self.txinindex)?;
vecs.height_to_first_txoutindex
.push_if_needed(height, self.txoutindex)?;
vecs.height_to_first_addressindex
@@ -54,8 +57,10 @@ impl Indexes {
.push_if_needed(height, self.p2pk65index)?;
vecs.height_to_first_p2pkhindex
.push_if_needed(height, self.p2pkhindex)?;
vecs.height_to_first_p2shindex.push_if_needed(height, self.p2shindex)?;
vecs.height_to_first_p2trindex.push_if_needed(height, self.p2trindex)?;
vecs.height_to_first_p2shindex
.push_if_needed(height, self.p2shindex)?;
vecs.height_to_first_p2trindex
.push_if_needed(height, self.p2trindex)?;
vecs.height_to_first_p2wpkhindex
.push_if_needed(height, self.p2wpkhindex)?;
vecs.height_to_first_p2wshindex
@@ -77,9 +82,11 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
// Height at which we wanna start: min last saved + 1 or 0
let starting_height = vecs.starting_height().min(stores.starting_height());
let range = starting_height
.checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32)
.unwrap_or_default()..*starting_height;
let range = u32::from(
starting_height
.checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32)
.unwrap_or_default(),
)..u32::from(starting_height);
// But we also need to check the chain and start earlier in case of a reorg
let height = range // ..= because of last saved + 1
@@ -107,8 +114,14 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?,
emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?,
height,
multisigindex: *vecs.height_to_first_multisigindex.get(height)?.context("")?,
opreturnindex: *vecs.height_to_first_opreturnindex.get(height)?.context("")?,
multisigindex: *vecs
.height_to_first_multisigindex
.get(height)?
.context("")?,
opreturnindex: *vecs
.height_to_first_opreturnindex
.get(height)?
.context("")?,
p2pk33index: *vecs.height_to_first_p2pk33index.get(height)?.context("")?,
p2pk65index: *vecs.height_to_first_p2pk65index.get(height)?.context("")?,
p2pkhindex: *vecs.height_to_first_p2pkhindex.get(height)?.context("")?,
@@ -116,7 +129,10 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
p2trindex: *vecs.height_to_first_p2trindex.get(height)?.context("")?,
p2wpkhindex: *vecs.height_to_first_p2wpkhindex.get(height)?.context("")?,
p2wshindex: *vecs.height_to_first_p2wshindex.get(height)?.context("")?,
pushonlyindex: *vecs.height_to_first_pushonlyindex.get(height)?.context("")?,
pushonlyindex: *vecs
.height_to_first_pushonlyindex
.get(height)?
.context("")?,
txindex: *vecs.height_to_first_txindex.get(height)?.context("")?,
txinindex: *vecs.height_to_first_txinindex.get(height)?.context("")?,
txoutindex: *vecs.height_to_first_txoutindex.get(height)?.context("")?,

View File

@@ -83,7 +83,7 @@ impl Indexer {
self.stores
.as_mut()
.unwrap()
.rollback_if_needed(self.vecs.as_ref().unwrap(), &starting_indexes)?;
.rollback_if_needed(self.vecs.as_mut().unwrap(), &starting_indexes)?;
self.vecs
.as_mut()
.unwrap()

View File

@@ -1,6 +1,9 @@
use std::{path::Path, thread};
use brk_core::{AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix, Txindex};
use brk_core::{
AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix,
Txindex,
};
use brk_vec::{Value, Version};
use crate::Indexes;
@@ -23,12 +26,14 @@ pub struct Stores {
impl Stores {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
thread::scope(|scope| {
let addresshash_to_addressindex =
scope.spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::from(1)));
let blockhash_prefix_to_height =
scope.spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1)));
let txid_prefix_to_txindex =
scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1)));
let addresshash_to_addressindex = scope.spawn(|| {
Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))
});
let blockhash_prefix_to_height = scope.spawn(|| {
Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))
});
let txid_prefix_to_txindex = scope
.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1)));
Ok(Self {
addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?,
@@ -38,22 +43,29 @@ impl Stores {
})
}
pub fn rollback_if_needed(&mut self, vecs: &Vecs, starting_indexes: &Indexes) -> color_eyre::Result<()> {
pub fn rollback_if_needed(
&mut self,
vecs: &mut Vecs,
starting_indexes: &Indexes,
) -> color_eyre::Result<()> {
vecs.height_to_blockhash
.iter_from(starting_indexes.height, |(_, blockhash)| {
.iter_from(starting_indexes.height, |(_, blockhash, ..)| {
let blockhash_prefix = BlockHashPrefix::from(blockhash);
self.blockhash_prefix_to_height.remove(blockhash_prefix);
Ok(())
})?;
vecs.txindex_to_txid
.iter_from(starting_indexes.txindex, |(_txindex, txid)| {
.iter_from(starting_indexes.txindex, |(_txindex, txid, ..)| {
let txid_prefix = TxidPrefix::from(txid);
self.txid_prefix_to_txindex.remove(txid_prefix);
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2pk65index
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65addressbytes
@@ -67,7 +79,10 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2pk33index.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2pk33index
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33addressbytes
@@ -81,9 +96,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2pkhindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2pkhindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2pkhindex_to_p2pkhaddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhaddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PKH));
self.addresshash_to_addressindex.remove(hash);
@@ -91,9 +113,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2shindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2shindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2shindex_to_p2shaddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2shindex_to_p2shaddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2SH));
self.addresshash_to_addressindex.remove(hash);
@@ -101,9 +130,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2trindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2trindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2trindex_to_p2traddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2trindex_to_p2traddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2TR));
self.addresshash_to_addressindex.remove(hash);
@@ -111,7 +147,10 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2wpkhindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2wpkhindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhaddressbytes
@@ -125,9 +164,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2wshindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2wshindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2wshindex_to_p2wshaddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2wshindex_to_p2wshaddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WSH));
self.addresshash_to_addressindex.remove(hash);
@@ -135,7 +181,7 @@ impl Stores {
}
}
self.commit(starting_indexes.height.decremented())?;
self.commit(starting_indexes.height.decremented().unwrap())?;
Ok(())
}
@@ -158,7 +204,8 @@ impl Stores {
scope.spawn(|| self.addresshash_to_addressindex.commit(height));
let blockhash_prefix_to_height_commit_handle =
scope.spawn(|| self.blockhash_prefix_to_height.commit(height));
let txid_prefix_to_txindex_commit_handle = scope.spawn(|| self.txid_prefix_to_txindex.commit(height));
let txid_prefix_to_txindex_commit_handle =
scope.spawn(|| self.txid_prefix_to_txindex.commit(height));
addresshash_to_addressindex_commit_handle.join().unwrap()?;
blockhash_prefix_to_height_commit_handle.join().unwrap()?;

View File

@@ -1,10 +1,11 @@
use std::{fs, io, path::Path};
use brk_core::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex,
Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex,
P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes,
P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height,
LockTime, Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes,
P2PK65index, P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes,
P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats,
Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
};
use brk_vec::{AnyStorableVec, Version};
use rayon::prelude::*;
@@ -75,9 +76,18 @@ impl Vecs {
&path.join("addressindex_to_addresstypeindex"),
Version::from(1),
)?,
addressindex_to_height: StorableVec::import(&path.join("addressindex_to_height"), Version::from(1))?,
height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?,
height_to_difficulty: StorableVec::import(&path.join("height_to_difficulty"), Version::from(1))?,
addressindex_to_height: StorableVec::import(
&path.join("addressindex_to_height"),
Version::from(1),
)?,
height_to_blockhash: StorableVec::import(
&path.join("height_to_blockhash"),
Version::from(1),
)?,
height_to_difficulty: StorableVec::import(
&path.join("height_to_difficulty"),
Version::from(1),
)?,
height_to_first_addressindex: StorableVec::import(
&path.join("height_to_first_addressindex"),
Version::from(1),
@@ -98,8 +108,14 @@ impl Vecs {
&path.join("height_to_first_pushonlyindex"),
Version::from(1),
)?,
height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"), Version::from(1))?,
height_to_first_txinindex: StorableVec::import(&path.join("height_to_first_txinindex"), Version::from(1))?,
height_to_first_txindex: StorableVec::import(
&path.join("height_to_first_txindex"),
Version::from(1),
)?,
height_to_first_txinindex: StorableVec::import(
&path.join("height_to_first_txinindex"),
Version::from(1),
)?,
height_to_first_txoutindex: StorableVec::import(
&path.join("height_to_first_txoutindex"),
Version::from(1),
@@ -120,8 +136,14 @@ impl Vecs {
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
)?,
height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?,
height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?,
height_to_first_p2shindex: StorableVec::import(
&path.join("height_to_first_p2shindex"),
Version::from(1),
)?,
height_to_first_p2trindex: StorableVec::import(
&path.join("height_to_first_p2trindex"),
Version::from(1),
)?,
height_to_first_p2wpkhindex: StorableVec::import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
@@ -131,8 +153,14 @@ impl Vecs {
Version::from(1),
)?,
height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?,
height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?,
height_to_timestamp: StorableVec::import(
&path.join("height_to_timestamp"),
Version::from(1),
)?,
height_to_weight: StorableVec::import(
&path.join("height_to_weight"),
Version::from(1),
)?,
p2pk33index_to_p2pk33addressbytes: StorableVec::import(
&path.join("p2pk33index_to_p2pk33addressbytes"),
Version::from(1),
@@ -169,27 +197,48 @@ impl Vecs {
&path.join("txindex_to_first_txoutindex"),
Version::from(1),
)?,
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_locktime: StorableVec::import(&path.join("txindex_to_locktime"), Version::from(1))?,
txindex_to_height: StorableVec::import(
&path.join("txindex_to_height"),
Version::from(1),
)?,
txindex_to_locktime: StorableVec::import(
&path.join("txindex_to_locktime"),
Version::from(1),
)?,
txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?,
txindex_to_base_size: StorableVec::import(&path.join("txindex_to_base_size"), Version::from(1))?,
txindex_to_total_size: StorableVec::import(&path.join("txindex_to_total_size"), Version::from(1))?,
txindex_to_base_size: StorableVec::import(
&path.join("txindex_to_base_size"),
Version::from(1),
)?,
txindex_to_total_size: StorableVec::import(
&path.join("txindex_to_total_size"),
Version::from(1),
)?,
txindex_to_is_explicitly_rbf: StorableVec::import(
&path.join("txindex_to_is_explicitly_rbf"),
Version::from(1),
)?,
txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?,
txinindex_to_txoutindex: StorableVec::import(&path.join("txinindex_to_txoutindex"), Version::from(1))?,
txindex_to_txversion: StorableVec::import(
&path.join("txindex_to_txversion"),
Version::from(1),
)?,
txinindex_to_txoutindex: StorableVec::import(
&path.join("txinindex_to_txoutindex"),
Version::from(1),
)?,
txoutindex_to_addressindex: StorableVec::import(
&path.join("txoutindex_to_addressindex"),
Version::from(1),
)?,
txoutindex_to_value: StorableVec::import(&path.join("txoutindex_to_value"), Version::from(1))?,
txoutindex_to_value: StorableVec::import(
&path.join("txoutindex_to_value"),
Version::from(1),
)?,
})
}
pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> {
let saved_height = starting_indexes.height.decremented();
let saved_height = starting_indexes.height.decremented().unwrap_or_default();
// We don't want to override the starting indexes so we cut from n + 1
let height = starting_indexes.height.incremented();
@@ -218,7 +267,8 @@ impl Vecs {
.truncate_if_needed(height, saved_height)?;
self.height_to_first_pushonlyindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txindex.truncate_if_needed(height, saved_height)?;
self.height_to_first_txindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txinindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txoutindex
@@ -243,11 +293,16 @@ impl Vecs {
..
} = starting_indexes;
self.height_to_blockhash.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty.truncate_if_needed(height, saved_height)?;
self.height_to_size.truncate_if_needed(height, saved_height)?;
self.height_to_timestamp.truncate_if_needed(height, saved_height)?;
self.height_to_weight.truncate_if_needed(height, saved_height)?;
self.height_to_blockhash
.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty
.truncate_if_needed(height, saved_height)?;
self.height_to_size
.truncate_if_needed(height, saved_height)?;
self.height_to_timestamp
.truncate_if_needed(height, saved_height)?;
self.height_to_weight
.truncate_if_needed(height, saved_height)?;
self.addressindex_to_addresstype
.truncate_if_needed(addressindex, saved_height)?;
@@ -275,12 +330,18 @@ impl Vecs {
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_first_txoutindex
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_height.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_locktime.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txid.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txversion.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_base_size.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_total_size.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_height
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_locktime
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txid
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txversion
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_base_size
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_total_size
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_is_explicitly_rbf
.truncate_if_needed(txindex, saved_height)?;
@@ -289,7 +350,8 @@ impl Vecs {
self.txoutindex_to_addressindex
.truncate_if_needed(txoutindex, saved_height)?;
self.txoutindex_to_value.truncate_if_needed(txoutindex, saved_height)?;
self.txoutindex_to_value
.truncate_if_needed(txoutindex, saved_height)?;
Ok(())
}
@@ -351,13 +413,21 @@ impl Vecs {
Addressbytes::P2PK33(bytes) => self
.p2pk33index_to_p2pk33addressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2PKH(bytes) => self
.p2pkhindex_to_p2pkhaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2SH(bytes) => self
.p2shindex_to_p2shaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2WPKH(bytes) => self
.p2wpkhindex_to_p2wpkhaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2WSH(bytes) => self
.p2wshindex_to_p2wshaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2TR(bytes) => self
.p2trindex_to_p2traddressbytes
.push_if_needed(index.into(), bytes),
}
}