Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-14 20:48:12 -07:00
Code Issues Packages Projects Releases Wiki Activity

indexer: improved rollback; global: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-18 22:43:12 +01:00
parent a122333aaa
commit 15f2e05192
29 changed files with 708 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
indexer/src/lib.rs
+59 -103
View File
@@ -22,7 +22,6 @@ pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta};
use storage::{Fjalls, StorableVecs};
pub use structs::*;
const UNSAFE_BLOCKS: u32 = 1000;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
pub struct Indexer<const MODE: u8> {
@@ -43,68 +42,42 @@ impl Indexer<CACHED_GETS> {
pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
let check_collisions = true;
let vecs = &mut self.vecs;
let trees = &mut self.trees;
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.trees, &rpc)).unwrap_or_else(|_| {
let indexes = Indexes::default();
indexes.push_if_needed(&mut self.vecs).unwrap();
indexes
});
let mut height = vecs
.min_height()
.unwrap_or_default()
.min(trees.min_height())
.and_then(|h| h.checked_sub(UNSAFE_BLOCKS))
.map(Height::from)
.unwrap_or_default();
let mut txindex_global = vecs.height_to_first_txindex.get_or_default(height)?;
let mut txinindex_global = vecs.height_to_first_txinindex.get_or_default(height)?;
let mut txoutindex_global = vecs.height_to_first_txoutindex.get_or_default(height)?;
let mut addressindex_global = vecs.height_to_first_addressindex.get_or_default(height)?;
let mut emptyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut multisigindex_global = vecs.height_to_first_multisigindex.get_or_default(height)?;
let mut opreturnindex_global = vecs.height_to_first_opreturnindex.get_or_default(height)?;
let mut pushonlyindex_global = vecs.height_to_first_pushonlyindex.get_or_default(height)?;
let mut unknownindex_global = vecs.height_to_first_unknownindex.get_or_default(height)?;
let mut p2pk33index_global = vecs.height_to_first_p2pk33index.get_or_default(height)?;
let mut p2pk65index_global = vecs.height_to_first_p2pk65index.get_or_default(height)?;
let mut p2pkhindex_global = vecs.height_to_first_p2pkhindex.get_or_default(height)?;
let mut p2shindex_global = vecs.height_to_first_p2shindex.get_or_default(height)?;
let mut p2trindex_global = vecs.height_to_first_p2trindex.get_or_default(height)?;
let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
exit.block();
self.trees.rollback(&self.vecs, &starting_indexes)?;
self.vecs.rollback(&starting_indexes)?;
exit.unblock();
let export =
|trees: &mut Fjalls, vecs: &mut StorableVecs<CACHED_GETS>, height: Height| -> color_eyre::Result<()> {
info!("Exporting...");
exit.block();
thread::scope(|scope| -> color_eyre::Result<()> {
let vecs_handle = scope.spawn(|| vecs.flush(height));
trees.commit(height)?;
vecs_handle.join().unwrap()?;
Ok(())
})?;
trees.commit(height)?;
vecs.flush(height)?;
exit.unblock();
Ok(())
};
iterator::new(bitcoin_dir, Some(height.into()), Some(400_000), rpc)
let vecs = &mut self.vecs;
let trees = &mut self.trees;
let mut idxs = starting_indexes;
iterator::new(bitcoin_dir, Some(idxs.height.into()), None, rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {_height}...");
let height = Height::from(_height);
idxs.height = height;
let blockhash = BlockHash::from(blockhash);
height = Height::from(_height);
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
if &blockhash != saved_blockhash.as_ref() {
todo!("Rollback not implemented");
// trees.rollback_from(&mut rtx, height, &exit)?;
}
}
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
let blockhash_prefix = BlockHashPrefix::from(&blockhash);
if trees
.blockhash_prefix_to_height
@@ -124,30 +97,6 @@ impl Indexer<CACHED_GETS> {
vecs.height_to_timestamp.push_if_needed(height, Timestamp::from(block.header.time))?;
vecs.height_to_size.push_if_needed(height, block.total_size())?;
vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
vecs.height_to_first_txindex.push_if_needed(height, txindex_global)?;
vecs.height_to_first_txinindex
.push_if_needed(height, txinindex_global)?;
vecs.height_to_first_txoutindex
.push_if_needed(height, txoutindex_global)?;
vecs.height_to_first_addressindex
.push_if_needed(height, addressindex_global)?;
vecs.height_to_first_emptyindex
.push_if_needed(height, emptyindex_global)?;
vecs.height_to_first_multisigindex
.push_if_needed(height, multisigindex_global)?;
vecs.height_to_first_opreturnindex
.push_if_needed(height, opreturnindex_global)?;
vecs.height_to_first_pushonlyindex
.push_if_needed(height, pushonlyindex_global)?;
vecs.height_to_first_unknownindex
.push_if_needed(height, unknownindex_global)?;
vecs.height_to_first_p2pk33index.push_if_needed(height, p2pk33index_global)?;
vecs.height_to_first_p2pk65index.push_if_needed(height, p2pk65index_global)?;
vecs.height_to_first_p2pkhindex.push_if_needed(height, p2pkhindex_global)?;
vecs.height_to_first_p2shindex.push_if_needed(height, p2shindex_global)?;
vecs.height_to_first_p2trindex.push_if_needed(height, p2trindex_global)?;
vecs.height_to_first_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?;
vecs.height_to_first_p2wshindex.push_if_needed(height, p2wshindex_global)?;
let inputs = block
.txdata
@@ -191,7 +140,7 @@ impl Indexer<CACHED_GETS> {
.map(|(index, tx)| -> color_eyre::Result<_> {
let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::try_from(&txid)?;
let txid_prefix = TxidPrefix::from(&txid);
let prev_txindex_opt =
if check_collisions && trees.txid_prefix_to_txindex.needs(height) {
@@ -224,8 +173,8 @@ impl Indexer<CACHED_GETS> {
.into_par_iter()
.enumerate()
.map(|(block_txinindex, (block_txindex, vin, txin, tx))| -> color_eyre::Result<(Txinindex, InputSource)> {
let txindex = txindex_global + block_txindex;
let txinindex = txinindex_global + Txinindex::from(block_txinindex);
let txindex = idxs.txindex + block_txindex;
let txinindex = idxs.txinindex + Txinindex::from(block_txinindex);
// dbg!((txindex, txinindex, vin));
@@ -238,15 +187,15 @@ impl Indexer<CACHED_GETS> {
let prev_txindex = if let Some(txindex) = trees
.txid_prefix_to_txindex
.get(&TxidPrefix::try_from(&txid)?)?
.get(&TxidPrefix::from(&txid))?
.map(|v| *v)
.and_then(|txindex| {
// Checking if not finding txindex from the future
(txindex < txindex_global).then_some(txindex)
(txindex < idxs.txindex).then_some(txindex)
}) {
txindex
} else {
// dbg!(txindex_global + block_txindex, txindex, txin, vin);
// dbg!(indexes.txindex + block_txindex, txindex, txin, vin);
return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
};
@@ -301,8 +250,8 @@ impl Indexer<CACHED_GETS> {
&Transaction,
),
)> {
let txindex = txindex_global + block_txindex;
let txoutindex = txoutindex_global + Txoutindex::from(block_txoutindex);
let txindex = idxs.txindex + block_txindex;
let txoutindex = idxs.txoutindex + Txoutindex::from(block_txoutindex);
let script = &txout.script_pubkey;
@@ -321,7 +270,7 @@ impl Indexer<CACHED_GETS> {
.map(|v| *v)
// Checking if not in the future
.and_then(|addressindex_local| {
(addressindex_local < addressindex_global).then_some(addressindex_local)
(addressindex_local < idxs.addressindex).then_some(addressindex_local)
})
});
@@ -359,7 +308,7 @@ impl Indexer<CACHED_GETS> {
prev_addresstype,
prev_addressbytes,
addressbytes,
addressindex_global,
idxs.addressindex,
addressindex,
addresstypeindex,
txout,
@@ -448,7 +397,7 @@ impl Indexer<CACHED_GETS> {
vecs.txoutindex_to_value.push_if_needed(txoutindex, sats)?;
let mut addressindex = addressindex_global;
let mut addressindex = idxs.addressindex;
let mut addresshash = None;
@@ -464,21 +413,21 @@ impl Indexer<CACHED_GETS> {
}) {
addressindex = addressindex_local;
} else {
addressindex_global.increment();
idxs.addressindex.increment();
let addresstypeindex = match addresstype {
Addresstype::Empty => emptyindex_global.copy_then_increment(),
Addresstype::Multisig => multisigindex_global.copy_then_increment(),
Addresstype::OpReturn => opreturnindex_global.copy_then_increment(),
Addresstype::PushOnly => pushonlyindex_global.copy_then_increment(),
Addresstype::Unknown => unknownindex_global.copy_then_increment(),
Addresstype::P2PK65 => p2pk65index_global.copy_then_increment(),
Addresstype::P2PK33 => p2pk33index_global.copy_then_increment(),
Addresstype::P2PKH => p2pkhindex_global.copy_then_increment(),
Addresstype::P2SH => p2shindex_global.copy_then_increment(),
Addresstype::P2WPKH => p2wpkhindex_global.copy_then_increment(),
Addresstype::P2WSH => p2wshindex_global.copy_then_increment(),
Addresstype::P2TR => p2trindex_global.copy_then_increment(),
Addresstype::Empty => idxs.emptyindex.copy_then_increment(),
Addresstype::Multisig => idxs.multisigindex.copy_then_increment(),
Addresstype::OpReturn => idxs.opreturnindex.copy_then_increment(),
Addresstype::PushOnly => idxs.pushonlyindex.copy_then_increment(),
Addresstype::Unknown => idxs.unknownindex.copy_then_increment(),
Addresstype::P2PK65 => idxs.p2pk65index.copy_then_increment(),
Addresstype::P2PK33 => idxs.p2pk33index.copy_then_increment(),
Addresstype::P2PKH => idxs.p2pkhindex.copy_then_increment(),
Addresstype::P2SH => idxs.p2shindex.copy_then_increment(),
Addresstype::P2WPKH => idxs.p2wpkhindex.copy_then_increment(),
Addresstype::P2WSH => idxs.p2wshindex.copy_then_increment(),
Addresstype::P2TR => idxs.p2trindex.copy_then_increment(),
};
vecs.addressindex_to_addresstype
@@ -537,13 +486,13 @@ impl Indexer<CACHED_GETS> {
let vout = Vout::from(outpoint.vout);
let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.get(&TxidPrefix::try_from(&txid)?)
.get(&TxidPrefix::from(&txid))
.context("txid should be in same block").inspect_err(|_| {
dbg!(&txid_prefix_to_txid_and_block_txindex_and_prev_txindex);
// panic!();
})?
.2;
let prev_txindex = txindex_global + block_txindex;
let prev_txindex = idxs.txindex + block_txindex;
let prev_txoutindex = new_txindexvout_to_txoutindex
.remove(&(prev_txindex, vout))
@@ -577,7 +526,7 @@ impl Indexer<CACHED_GETS> {
.into_iter()
.try_for_each(
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
let txindex = txindex_global + index;
let txindex = idxs.txindex + index;
txindex_to_tx_and_txid.insert(txindex, (tx, txid));
@@ -640,12 +589,19 @@ impl Indexer<CACHED_GETS> {
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
vecs.txindex_to_height.push_if_needed(txindex, height)?;
vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time.into())?;
// tx.base_size()
// tx.total_size()
// tx.is_explicitly_rbf()
// tx.weight()
// tx.vsize() in computer as it can be computed from the weight
Ok(())
})?;
txindex_global += Txindex::from(tx_len);
txinindex_global += Txinindex::from(inputs_len);
txoutindex_global += Txoutindex::from(outputs_len);
idxs.txindex += Txindex::from(tx_len);
idxs.txinindex += Txinindex::from(inputs_len);
idxs.txoutindex += Txoutindex::from(outputs_len);
idxs.push_future_if_needed(vecs)?;
let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked();
if should_snapshot {
@@ -655,7 +611,7 @@ impl Indexer<CACHED_GETS> {
Ok(())
})?;
export(trees, vecs, height)?;
export(trees, vecs, idxs.height)?;
sleep(Duration::from_millis(100));
indexer/src/storage/fjalls/base.rs
+37 -6
View File
@@ -1,4 +1,8 @@
use std::{collections::BTreeMap, error, mem, path::Path};
use std::{
collections::{BTreeMap, BTreeSet},
error, mem,
path::Path,
};
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
@@ -17,8 +21,11 @@ pub struct Store<Key, Value> {
part: TransactionalPartitionHandle,
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
}
const CHECK_COLLISISONS: bool = true;
impl<K, V> Store<K, V>
where
K: Into<Slice> + Ord + Immutable + IntoBytes,
@@ -48,6 +55,7 @@ where
part,
rtx,
puts: BTreeMap::new(),
dels: BTreeSet::new(),
})
}
@@ -63,21 +71,44 @@ where
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
if !self.dels.is_empty() {
unreachable!("Shouldn't reach this");
// self.dels.remove(&key);
}
self.puts.insert(key, value);
}
}
pub fn remove(&mut self, key: K) {
if !self.puts.is_empty() {
unreachable!("Shouldn't reach this");
// self.puts.remove(&key);
}
self.dels.insert(key);
}
pub fn commit(&mut self, height: Height) -> Result<()> {
if self.has(height) && self.puts.is_empty() {
if self.has(height) && self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
}
self.meta.export(self.len(), height)?;
let mut wtx = self.keyspace.write_tx();
mem::take(&mut self.puts)
mem::take(&mut self.dels)
.into_iter()
.for_each(|(key, value)| wtx.insert(&self.part, key, value));
.for_each(|key| wtx.remove(&self.part, key));
mem::take(&mut self.puts).into_iter().for_each(|(key, value)| {
if CHECK_COLLISISONS {
if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) {
dbg!(value);
unreachable!();
}
}
wtx.insert(&self.part, key, value)
});
wtx.commit()?;
@@ -88,12 +119,12 @@ where
Ok(())
}
pub fn height(&self) -> Option<&Height> {
pub fn height(&self) -> Option<Height> {
self.meta.height()
}
pub fn len(&self) -> usize {
self.meta.len() + self.puts.len()
self.meta.len() + self.puts.len() - self.dels.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
indexer/src/storage/fjalls/meta.rs
+2 -2
View File
@@ -67,8 +67,8 @@ impl StoreMeta {
path.join("version")
}
pub fn height(&self) -> Option<&Height> {
self.height.as_ref()
pub fn height(&self) -> Option<Height> {
self.height
}
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)
indexer/src/storage/fjalls/mod.rs
+117 -132
View File
@@ -1,16 +1,19 @@
use std::{path::Path, thread};
use storable_vec::Version;
use storable_vec::{Value, Version, CACHED_GETS};
use crate::{AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
use crate::{
AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, Indexes, TxidPrefix, Txindex,
};
mod base;
mod meta;
// mod version;
pub use base::*;
pub use meta::*;
use super::StorableVecs;
pub struct Fjalls {
pub addresshash_to_addressindex: Store<AddressHash, Addressindex>,
pub blockhash_prefix_to_height: Store<BlockHashPrefix, Height>,
@@ -30,135 +33,135 @@ impl Fjalls {
})
}
// pub fn rollback_from(
// &mut self,
// _wtx: &mut WriteTransaction,
// _height: Height,
// _exit: &Exit,
// ) -> color_eyre::Result<()> {
// panic!();
// let mut txindex = None;
pub fn rollback(&mut self, vecs: &StorableVecs<CACHED_GETS>, starting_indexes: &Indexes) -> color_eyre::Result<()> {
vecs.height_to_blockhash
.iter_from(starting_indexes.height, |(_, blockhash)| {
let blockhash = blockhash.as_ref();
let blockhash_prefix = BlockHashPrefix::from(blockhash);
self.blockhash_prefix_to_height.remove(blockhash_prefix);
Ok(())
})?;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
vecs.txindex_to_txid.iter_from(starting_indexes.txindex, |(_, txid)| {
let txid = txid.as_ref();
let txid_prefix = TxidPrefix::from(txid);
self.txid_prefix_to_txindex.remove(txid_prefix);
Ok(())
})?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
vecs.height_to_first_p2pk65index
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65addressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK65));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
vecs.height_to_first_p2pk33index
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33addressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK33));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
vecs.height_to_first_p2pkhindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PKH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// Ok(())
// })?;
vecs.height_to_first_p2shindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2shindex_to_p2shaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2SH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// let txindex = txindex.context("txindex to not be none by now")?;
vecs.height_to_first_p2trindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2trindex_to_p2traddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2TR));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
vecs.height_to_first_p2wpkhindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WPKH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
vecs.height_to_first_p2wshindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2wshindex_to_p2wshaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WSH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// Ok(())
// })?;
self.commit(starting_indexes.height.decremented())?;
// let txoutindex = Txoutindex::from(txindex);
Ok(())
}
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// todo!("clear txindexvout_to_txoutindex")
// Ok(())
// }
pub fn min_height(&self) -> Option<Height> {
pub fn starting_height(&self) -> Height {
[
self.addresshash_to_addressindex.height(),
self.blockhash_prefix_to_height.height(),
self.txid_prefix_to_txindex.height(),
]
.into_iter()
.map(|height| height.map(Height::incremented).unwrap_or_default())
.min()
.flatten()
.cloned()
.unwrap()
}
pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
@@ -176,22 +179,4 @@ impl Fjalls {
Ok(())
})
}
// pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) {
// self.addressbytes_prefix_to_addressindex.update_meta(wtx, height);
// self.blockhash_prefix_to_height.update_meta(wtx, height);
// self.txid_prefix_to_txindex.update_meta(wtx, height);
// }
// pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
// thread::scope(|scope| {
// vec![
// scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
// scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
// scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
// ]
// .into_iter()
// .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
// })
// }
}
indexer/src/storage/storable_vecs/base.rs
+4 -4
View File
@@ -28,14 +28,14 @@ where
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
if self.needs(height) {
height.write(&self.path_height())?;
}
height.write(&self.path_height())?;
self.vec.flush()
}
pub fn truncate_if_needed(&mut self, index: I, height: Height) -> storable_vec::Result<Option<T>> {
height.write(&self.path_height())?;
if self.height.is_none_or(|self_height| self_height != height) {
height.write(&self.path_height())?;
}
self.vec.truncate_if_needed(index)
}
indexer/src/storage/storable_vecs/mod.rs
+101 -109
View File
@@ -1,14 +1,17 @@
use std::{collections::BTreeMap, fs, io, path::Path};
use std::{fs, io, path::Path};
use exit::Exit;
use rayon::prelude::*;
use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS};
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex,
Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex,
P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes,
P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
use crate::{
structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime,
Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index,
P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes,
P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex,
Txinindex, Txoutindex, Unknownindex, Weight,
},
Indexes,
};
mod base;
@@ -58,8 +61,6 @@ pub struct StorableVecs<const MODE: u8> {
pub txoutindex_to_value: StorableVec<Txoutindex, Sats, MODE>,
}
// const UNSAFE_BLOCKS: usize = 1000;
impl<const MODE: u8> StorableVecs<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -180,113 +181,104 @@ impl<const MODE: u8> StorableVecs<MODE> {
})
}
#[allow(unused)]
pub fn rollback_from(&mut self, height: Height, exit: &Exit) -> storable_vec::Result<()> {
let prev_height = height.decremented();
pub fn rollback(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> {
let saved_height = starting_indexes.height.decremented();
let mut truncated_indexes: BTreeMap<String, Option<usize>> = BTreeMap::new();
// We don't want to override the starting indexes so we cut from n + 1
let height = starting_indexes.height.incremented();
let addressindex = self
.height_to_first_addressindex
.truncate_if_needed(height, prev_height)?;
let txindex = self.height_to_first_txindex.truncate_if_needed(height, prev_height)?;
let txinindex = self.height_to_first_txinindex.truncate_if_needed(height, prev_height)?;
let txoutindex = self
.height_to_first_txoutindex
.truncate_if_needed(height, prev_height)?;
let p2pk33index = self
.height_to_first_p2pk33index
.truncate_if_needed(height, prev_height)?;
let p2pk65index = self
.height_to_first_p2pk65index
.truncate_if_needed(height, prev_height)?;
let p2pkhindex = self
.height_to_first_p2pkhindex
.truncate_if_needed(height, prev_height)?;
let p2shindex = self.height_to_first_p2shindex.truncate_if_needed(height, prev_height)?;
let p2trindex = self.height_to_first_p2trindex.truncate_if_needed(height, prev_height)?;
let p2wpkhindex = self
.height_to_first_p2wpkhindex
.truncate_if_needed(height, prev_height)?;
let p2wshindex = self
.height_to_first_p2wshindex
.truncate_if_needed(height, prev_height)?;
self.height_to_blockhash.truncate_if_needed(height, prev_height)?;
self.height_to_difficulty.truncate_if_needed(height, prev_height)?;
self.height_to_first_addressindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_emptyindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_multisigindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_opreturnindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2pk33index
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2pk65index
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2pkhindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2shindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2trindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2wpkhindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2wshindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_pushonlyindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txindex.truncate_if_needed(height, saved_height)?;
self.height_to_first_txinindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txoutindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_unknownindex
.truncate_if_needed(height, prev_height)?;
self.height_to_size.truncate_if_needed(height, prev_height)?;
self.height_to_timestamp.truncate_if_needed(height, prev_height)?;
self.height_to_weight.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
if let Some(addressindex) = addressindex {
self.addressindex_to_addresstype
.truncate_if_needed(addressindex, prev_height)?;
self.addressindex_to_addresstypeindex
.truncate_if_needed(addressindex, prev_height)?;
self.addressindex_to_height
.truncate_if_needed(addressindex, prev_height)?;
}
// Now we can cut everything that's out of date
let &Indexes {
addressindex,
height,
p2pk33index,
p2pk65index,
p2pkhindex,
p2shindex,
p2trindex,
p2wpkhindex,
p2wshindex,
txindex,
txinindex,
txoutindex,
..
} = starting_indexes;
if let Some(p2pk33index) = p2pk33index {
self.p2pk33index_to_p2pk33addressbytes
.truncate_if_needed(p2pk33index, prev_height)?;
}
if let Some(p2pk65index) = p2pk65index {
self.p2pk65index_to_p2pk65addressbytes
.truncate_if_needed(p2pk65index, prev_height)?;
}
if let Some(p2pkhindex) = p2pkhindex {
self.p2pkhindex_to_p2pkhaddressbytes
.truncate_if_needed(p2pkhindex, prev_height)?;
}
if let Some(p2shindex) = p2shindex {
self.p2shindex_to_p2shaddressbytes
.truncate_if_needed(p2shindex, prev_height)?;
}
if let Some(p2trindex) = p2trindex {
self.p2trindex_to_p2traddressbytes
.truncate_if_needed(p2trindex, prev_height)?;
}
if let Some(p2wpkhindex) = p2wpkhindex {
self.p2wpkhindex_to_p2wpkhaddressbytes
.truncate_if_needed(p2wpkhindex, prev_height)?;
}
if let Some(p2wshindex) = p2wshindex {
self.p2wshindex_to_p2wshaddressbytes
.truncate_if_needed(p2wshindex, prev_height);
}
self.height_to_blockhash.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty.truncate_if_needed(height, saved_height)?;
self.height_to_size.truncate_if_needed(height, saved_height)?;
self.height_to_timestamp.truncate_if_needed(height, saved_height)?;
self.height_to_weight.truncate_if_needed(height, saved_height)?;
if let Some(txindex) = txindex {
self.txindex_to_first_txinindex
.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_first_txoutindex
.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_height.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_locktime.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_txid.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_txversion.truncate_if_needed(txindex, prev_height)?;
}
self.addressindex_to_addresstype
.truncate_if_needed(addressindex, saved_height)?;
self.addressindex_to_addresstypeindex
.truncate_if_needed(addressindex, saved_height)?;
self.addressindex_to_height
.truncate_if_needed(addressindex, saved_height)?;
if let Some(txinindex) = txinindex {
self.txinindex_to_txoutindex
.truncate_if_needed(txinindex, prev_height)?;
}
self.p2pk33index_to_p2pk33addressbytes
.truncate_if_needed(p2pk33index, saved_height)?;
self.p2pk65index_to_p2pk65addressbytes
.truncate_if_needed(p2pk65index, saved_height)?;
self.p2pkhindex_to_p2pkhaddressbytes
.truncate_if_needed(p2pkhindex, saved_height)?;
self.p2shindex_to_p2shaddressbytes
.truncate_if_needed(p2shindex, saved_height)?;
self.p2trindex_to_p2traddressbytes
.truncate_if_needed(p2trindex, saved_height)?;
self.p2wpkhindex_to_p2wpkhaddressbytes
.truncate_if_needed(p2wpkhindex, saved_height)?;
self.p2wshindex_to_p2wshaddressbytes
.truncate_if_needed(p2wshindex, saved_height)?;
if let Some(txoutindex) = txoutindex {
self.txoutindex_to_addressindex
.truncate_if_needed(txoutindex, prev_height)?;
self.txoutindex_to_value.truncate_if_needed(txoutindex, prev_height)?;
}
self.txindex_to_first_txinindex
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_first_txoutindex
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_height.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_locktime.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txid.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txversion.truncate_if_needed(txindex, saved_height)?;
self.txinindex_to_txoutindex
.truncate_if_needed(txinindex, saved_height)?;
self.txoutindex_to_addressindex
.truncate_if_needed(txoutindex, saved_height)?;
self.txoutindex_to_value.truncate_if_needed(txoutindex, saved_height)?;
Ok(())
}
@@ -297,12 +289,12 @@ impl<const MODE: u8> StorableVecs<MODE> {
.try_for_each(|vec| vec.flush(height))
}
pub fn min_height(&mut self) -> color_eyre::Result<Option<Height>> {
Ok(self
.as_mut_any_vec_slice()
pub fn starting_height(&mut self) -> Height {
self.as_mut_any_vec_slice()
.into_iter()
.map(|vec| vec.height().unwrap_or_default())
.min())
.map(|vec| vec.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 40] {
indexer/src/structs/blockhash.rs
+10
View File
@@ -1,9 +1,12 @@
use std::mem;
use derive_deref::Deref;
use iterator::rpc::{Client, RpcApi};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct BlockHash([u8; 32]);
@@ -18,3 +21,10 @@ impl From<BlockHash> for bitcoin::BlockHash {
unsafe { mem::transmute(value) }
}
}
impl TryFrom<(&Client, Height)> for BlockHash {
type Error = iterator::rpc::Error;
fn try_from((rpc, height): (&Client, Height)) -> Result<Self, Self::Error> {
Ok(Self::from(rpc.get_block_hash(u64::from(height))?))
}
}
indexer/src/structs/compressed.rs
+9 -11
View File
@@ -41,10 +41,9 @@ impl From<AddressHash> for Slice {
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct BlockHashPrefix([u8; 8]);
impl TryFrom<&BlockHash> for BlockHashPrefix {
type Error = color_eyre::Report;
fn try_from(value: &BlockHash) -> Result<Self, Self::Error> {
Ok(Self(copy_first_8bytes(&value[..])))
impl From<&BlockHash> for BlockHashPrefix {
fn from(value: &BlockHash) -> Self {
Self(copy_first_8bytes(&value[..]).unwrap())
}
}
impl TryFrom<Slice> for BlockHashPrefix {
@@ -66,10 +65,9 @@ impl From<BlockHashPrefix> for Slice {
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct TxidPrefix([u8; 8]);
impl TryFrom<&Txid> for TxidPrefix {
type Error = color_eyre::Report;
fn try_from(value: &Txid) -> Result<Self, Self::Error> {
Ok(Self(copy_first_8bytes(&value[..])))
impl From<&Txid> for TxidPrefix {
fn from(value: &Txid) -> Self {
Self(copy_first_8bytes(&value[..]).unwrap())
}
}
impl TryFrom<Slice> for TxidPrefix {
@@ -89,14 +87,14 @@ impl From<TxidPrefix> for Slice {
}
}
fn copy_first_8bytes(slice: &[u8]) -> [u8; 8] {
fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8], ()> {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
panic!("bad len");
return Err(());
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
buf
Ok(buf)
}
indexer/src/structs/height.rs
+25 -1
View File
@@ -30,13 +30,31 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
pub struct Height(u32);
impl Height {
const ZERO: Self = Height(0);
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.as_bytes())
}
pub fn decremented(&self) -> Self {
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(self.0 + 1)
}
pub fn decrement(&mut self) {
self.0 -= 1;
}
pub fn decremented(self) -> Self {
Self(self.0.checked_sub(1).unwrap_or_default())
}
pub fn is_zero(self) -> bool {
self == Self::ZERO
}
}
impl PartialEq<u64> for Height {
@@ -133,6 +151,12 @@ impl From<Height> for usize {
}
}
impl From<Height> for u64 {
fn from(value: Height) -> Self {
value.0 as u64
}
}
impl TryFrom<&Path> for Height {
type Error = storable_vec::Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
indexer/src/structs/indexes.rs
+114
View File
@@ -0,0 +1,114 @@
use color_eyre::eyre::ContextCompat;
use iterator::rpc::Client;
use iterator::NUMBER_OF_UNSAFE_BLOCKS;
use storable_vec::CACHED_GETS;
use crate::storage::{Fjalls, StorableVecs};
use super::{
Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex,
P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
};
#[derive(Debug, Default)]
pub struct Indexes {
pub addressindex: Addressindex,
pub emptyindex: Emptyindex,
pub height: Height,
pub multisigindex: Multisigindex,
pub opreturnindex: Opreturnindex,
pub p2pk33index: P2PK33index,
pub p2pk65index: P2PK65index,
pub p2pkhindex: P2PKHindex,
pub p2shindex: P2SHindex,
pub p2trindex: P2TRindex,
pub p2wpkhindex: P2WPKHindex,
pub p2wshindex: P2WSHindex,
pub pushonlyindex: Pushonlyindex,
pub txindex: Txindex,
pub txinindex: Txinindex,
pub txoutindex: Txoutindex,
pub unknownindex: Unknownindex,
}
impl Indexes {
pub fn push_if_needed(&self, vecs: &mut StorableVecs<CACHED_GETS>) -> storable_vec::Result<()> {
let height = self.height;
vecs.height_to_first_txindex.push_if_needed(height, self.txindex)?;
vecs.height_to_first_txinindex.push_if_needed(height, self.txinindex)?;
vecs.height_to_first_txoutindex
.push_if_needed(height, self.txoutindex)?;
vecs.height_to_first_addressindex
.push_if_needed(height, self.addressindex)?;
vecs.height_to_first_emptyindex
.push_if_needed(height, self.emptyindex)?;
vecs.height_to_first_multisigindex
.push_if_needed(height, self.multisigindex)?;
vecs.height_to_first_opreturnindex
.push_if_needed(height, self.opreturnindex)?;
vecs.height_to_first_pushonlyindex
.push_if_needed(height, self.pushonlyindex)?;
vecs.height_to_first_unknownindex
.push_if_needed(height, self.unknownindex)?;
vecs.height_to_first_p2pk33index
.push_if_needed(height, self.p2pk33index)?;
vecs.height_to_first_p2pk65index
.push_if_needed(height, self.p2pk65index)?;
vecs.height_to_first_p2pkhindex
.push_if_needed(height, self.p2pkhindex)?;
vecs.height_to_first_p2shindex.push_if_needed(height, self.p2shindex)?;
vecs.height_to_first_p2trindex.push_if_needed(height, self.p2trindex)?;
vecs.height_to_first_p2wpkhindex
.push_if_needed(height, self.p2wpkhindex)?;
vecs.height_to_first_p2wshindex
.push_if_needed(height, self.p2wshindex)?;
Ok(())
}
pub fn push_future_if_needed(&mut self, vecs: &mut StorableVecs<CACHED_GETS>) -> storable_vec::Result<()> {
self.height.increment();
self.push_if_needed(vecs)?;
self.height.decrement();
Ok(())
}
}
impl TryFrom<(&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)> for Indexes {
type Error = color_eyre::Report;
fn try_from((vecs, trees, rpc): (&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)) -> color_eyre::Result<Self> {
// Height at which we wanna start: min last saved + 1 or 0
let starting_height = vecs.starting_height().min(trees.starting_height());
// But we also need to check the chain and start earlier in case of a reorg
let height = (starting_height
.checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32)
.unwrap_or_default()..*starting_height) // ..= because of last saved + 1
.map(Height::from)
.find(|height| {
let rpc_blockhash = BlockHash::try_from((rpc, *height)).unwrap();
let saved_blockhash = vecs.height_to_blockhash.get(*height).unwrap().unwrap();
&rpc_blockhash != saved_blockhash.as_ref()
})
.unwrap_or(starting_height);
Ok(Self {
addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?,
emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?,
height,
multisigindex: *vecs.height_to_first_multisigindex.get(height)?.context("")?,
opreturnindex: *vecs.height_to_first_opreturnindex.get(height)?.context("")?,
p2pk33index: *vecs.height_to_first_p2pk33index.get(height)?.context("")?,
p2pk65index: *vecs.height_to_first_p2pk65index.get(height)?.context("")?,
p2pkhindex: *vecs.height_to_first_p2pkhindex.get(height)?.context("")?,
p2shindex: *vecs.height_to_first_p2shindex.get(height)?.context("")?,
p2trindex: *vecs.height_to_first_p2trindex.get(height)?.context("")?,
p2wpkhindex: *vecs.height_to_first_p2wpkhindex.get(height)?.context("")?,
p2wshindex: *vecs.height_to_first_p2wshindex.get(height)?.context("")?,
pushonlyindex: *vecs.height_to_first_pushonlyindex.get(height)?.context("")?,
txindex: *vecs.height_to_first_txindex.get(height)?.context("")?,
txinindex: *vecs.height_to_first_txinindex.get(height)?.context("")?,
txoutindex: *vecs.height_to_first_txoutindex.get(height)?.context("")?,
unknownindex: *vecs.height_to_first_unknownindex.get(height)?.context("")?,
})
}
}
indexer/src/structs/mod.rs
+2
View File
@@ -5,6 +5,7 @@ mod addresstypeindex;
mod blockhash;
mod compressed;
mod height;
mod indexes;
mod locktime;
mod sats;
mod timestamp;
@@ -24,6 +25,7 @@ pub use addresstypeindex::*;
pub use blockhash::*;
pub use compressed::*;
pub use height::*;
pub use indexes::*;
pub use locktime::*;
pub use sats::*;
pub use timestamp::*;