indexer: improved rollback; global: snapshot

This commit is contained in:
nym21
2025-02-18 22:43:12 +01:00
parent a122333aaa
commit 15f2e05192
29 changed files with 708 additions and 523 deletions

View File

@@ -1,5 +1,6 @@
[package]
name = "bindex"
description = "A bitcoin-core indexer"
version = "0.1.0"
edition = "2021"

19
indexer/README.md Normal file
View File

@@ -0,0 +1,19 @@
# Indexer
A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../iterator`) and creates a database of the vecs (`../storable_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can used in your Rust code.
The crate only stores the bare minimum to be self sufficient and not have to use an RPC client (except for scripts which are not stored). If you need more data, checkout `../computer` which uses the outputs from the indexer to compute a whole range of datasets.
The neat thing about using simple vecs to store data is that you can parse the outputed files with another programming language such as Python very simply, you'll find an example below.
## Usage
Rust: `src/main.rs`
Python: `../python/parse.py`
## Outputs
Vecs: `src/storage/storable_vecs/mod.rs`
Stores: `src/storage/fjalls/mod.rs`

View File

@@ -22,7 +22,6 @@ pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta};
use storage::{Fjalls, StorableVecs};
pub use structs::*;
const UNSAFE_BLOCKS: u32 = 1000;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
pub struct Indexer<const MODE: u8> {
@@ -43,68 +42,42 @@ impl Indexer<CACHED_GETS> {
pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
let check_collisions = true;
let vecs = &mut self.vecs;
let trees = &mut self.trees;
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.trees, &rpc)).unwrap_or_else(|_| {
let indexes = Indexes::default();
indexes.push_if_needed(&mut self.vecs).unwrap();
indexes
});
let mut height = vecs
.min_height()
.unwrap_or_default()
.min(trees.min_height())
.and_then(|h| h.checked_sub(UNSAFE_BLOCKS))
.map(Height::from)
.unwrap_or_default();
let mut txindex_global = vecs.height_to_first_txindex.get_or_default(height)?;
let mut txinindex_global = vecs.height_to_first_txinindex.get_or_default(height)?;
let mut txoutindex_global = vecs.height_to_first_txoutindex.get_or_default(height)?;
let mut addressindex_global = vecs.height_to_first_addressindex.get_or_default(height)?;
let mut emptyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut multisigindex_global = vecs.height_to_first_multisigindex.get_or_default(height)?;
let mut opreturnindex_global = vecs.height_to_first_opreturnindex.get_or_default(height)?;
let mut pushonlyindex_global = vecs.height_to_first_pushonlyindex.get_or_default(height)?;
let mut unknownindex_global = vecs.height_to_first_unknownindex.get_or_default(height)?;
let mut p2pk33index_global = vecs.height_to_first_p2pk33index.get_or_default(height)?;
let mut p2pk65index_global = vecs.height_to_first_p2pk65index.get_or_default(height)?;
let mut p2pkhindex_global = vecs.height_to_first_p2pkhindex.get_or_default(height)?;
let mut p2shindex_global = vecs.height_to_first_p2shindex.get_or_default(height)?;
let mut p2trindex_global = vecs.height_to_first_p2trindex.get_or_default(height)?;
let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
exit.block();
self.trees.rollback(&self.vecs, &starting_indexes)?;
self.vecs.rollback(&starting_indexes)?;
exit.unblock();
let export =
|trees: &mut Fjalls, vecs: &mut StorableVecs<CACHED_GETS>, height: Height| -> color_eyre::Result<()> {
info!("Exporting...");
exit.block();
thread::scope(|scope| -> color_eyre::Result<()> {
let vecs_handle = scope.spawn(|| vecs.flush(height));
trees.commit(height)?;
vecs_handle.join().unwrap()?;
Ok(())
})?;
trees.commit(height)?;
vecs.flush(height)?;
exit.unblock();
Ok(())
};
iterator::new(bitcoin_dir, Some(height.into()), Some(400_000), rpc)
let vecs = &mut self.vecs;
let trees = &mut self.trees;
let mut idxs = starting_indexes;
iterator::new(bitcoin_dir, Some(idxs.height.into()), None, rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {_height}...");
let height = Height::from(_height);
idxs.height = height;
let blockhash = BlockHash::from(blockhash);
height = Height::from(_height);
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
if &blockhash != saved_blockhash.as_ref() {
todo!("Rollback not implemented");
// trees.rollback_from(&mut rtx, height, &exit)?;
}
}
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
let blockhash_prefix = BlockHashPrefix::from(&blockhash);
if trees
.blockhash_prefix_to_height
@@ -124,30 +97,6 @@ impl Indexer<CACHED_GETS> {
vecs.height_to_timestamp.push_if_needed(height, Timestamp::from(block.header.time))?;
vecs.height_to_size.push_if_needed(height, block.total_size())?;
vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
vecs.height_to_first_txindex.push_if_needed(height, txindex_global)?;
vecs.height_to_first_txinindex
.push_if_needed(height, txinindex_global)?;
vecs.height_to_first_txoutindex
.push_if_needed(height, txoutindex_global)?;
vecs.height_to_first_addressindex
.push_if_needed(height, addressindex_global)?;
vecs.height_to_first_emptyindex
.push_if_needed(height, emptyindex_global)?;
vecs.height_to_first_multisigindex
.push_if_needed(height, multisigindex_global)?;
vecs.height_to_first_opreturnindex
.push_if_needed(height, opreturnindex_global)?;
vecs.height_to_first_pushonlyindex
.push_if_needed(height, pushonlyindex_global)?;
vecs.height_to_first_unknownindex
.push_if_needed(height, unknownindex_global)?;
vecs.height_to_first_p2pk33index.push_if_needed(height, p2pk33index_global)?;
vecs.height_to_first_p2pk65index.push_if_needed(height, p2pk65index_global)?;
vecs.height_to_first_p2pkhindex.push_if_needed(height, p2pkhindex_global)?;
vecs.height_to_first_p2shindex.push_if_needed(height, p2shindex_global)?;
vecs.height_to_first_p2trindex.push_if_needed(height, p2trindex_global)?;
vecs.height_to_first_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?;
vecs.height_to_first_p2wshindex.push_if_needed(height, p2wshindex_global)?;
let inputs = block
.txdata
@@ -191,7 +140,7 @@ impl Indexer<CACHED_GETS> {
.map(|(index, tx)| -> color_eyre::Result<_> {
let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::try_from(&txid)?;
let txid_prefix = TxidPrefix::from(&txid);
let prev_txindex_opt =
if check_collisions && trees.txid_prefix_to_txindex.needs(height) {
@@ -224,8 +173,8 @@ impl Indexer<CACHED_GETS> {
.into_par_iter()
.enumerate()
.map(|(block_txinindex, (block_txindex, vin, txin, tx))| -> color_eyre::Result<(Txinindex, InputSource)> {
let txindex = txindex_global + block_txindex;
let txinindex = txinindex_global + Txinindex::from(block_txinindex);
let txindex = idxs.txindex + block_txindex;
let txinindex = idxs.txinindex + Txinindex::from(block_txinindex);
// dbg!((txindex, txinindex, vin));
@@ -238,15 +187,15 @@ impl Indexer<CACHED_GETS> {
let prev_txindex = if let Some(txindex) = trees
.txid_prefix_to_txindex
.get(&TxidPrefix::try_from(&txid)?)?
.get(&TxidPrefix::from(&txid))?
.map(|v| *v)
.and_then(|txindex| {
// Checking if not finding txindex from the future
(txindex < txindex_global).then_some(txindex)
(txindex < idxs.txindex).then_some(txindex)
}) {
txindex
} else {
// dbg!(txindex_global + block_txindex, txindex, txin, vin);
// dbg!(indexes.txindex + block_txindex, txindex, txin, vin);
return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
};
@@ -301,8 +250,8 @@ impl Indexer<CACHED_GETS> {
&Transaction,
),
)> {
let txindex = txindex_global + block_txindex;
let txoutindex = txoutindex_global + Txoutindex::from(block_txoutindex);
let txindex = idxs.txindex + block_txindex;
let txoutindex = idxs.txoutindex + Txoutindex::from(block_txoutindex);
let script = &txout.script_pubkey;
@@ -321,7 +270,7 @@ impl Indexer<CACHED_GETS> {
.map(|v| *v)
// Checking if not in the future
.and_then(|addressindex_local| {
(addressindex_local < addressindex_global).then_some(addressindex_local)
(addressindex_local < idxs.addressindex).then_some(addressindex_local)
})
});
@@ -359,7 +308,7 @@ impl Indexer<CACHED_GETS> {
prev_addresstype,
prev_addressbytes,
addressbytes,
addressindex_global,
idxs.addressindex,
addressindex,
addresstypeindex,
txout,
@@ -448,7 +397,7 @@ impl Indexer<CACHED_GETS> {
vecs.txoutindex_to_value.push_if_needed(txoutindex, sats)?;
let mut addressindex = addressindex_global;
let mut addressindex = idxs.addressindex;
let mut addresshash = None;
@@ -464,21 +413,21 @@ impl Indexer<CACHED_GETS> {
}) {
addressindex = addressindex_local;
} else {
addressindex_global.increment();
idxs.addressindex.increment();
let addresstypeindex = match addresstype {
Addresstype::Empty => emptyindex_global.copy_then_increment(),
Addresstype::Multisig => multisigindex_global.copy_then_increment(),
Addresstype::OpReturn => opreturnindex_global.copy_then_increment(),
Addresstype::PushOnly => pushonlyindex_global.copy_then_increment(),
Addresstype::Unknown => unknownindex_global.copy_then_increment(),
Addresstype::P2PK65 => p2pk65index_global.copy_then_increment(),
Addresstype::P2PK33 => p2pk33index_global.copy_then_increment(),
Addresstype::P2PKH => p2pkhindex_global.copy_then_increment(),
Addresstype::P2SH => p2shindex_global.copy_then_increment(),
Addresstype::P2WPKH => p2wpkhindex_global.copy_then_increment(),
Addresstype::P2WSH => p2wshindex_global.copy_then_increment(),
Addresstype::P2TR => p2trindex_global.copy_then_increment(),
Addresstype::Empty => idxs.emptyindex.copy_then_increment(),
Addresstype::Multisig => idxs.multisigindex.copy_then_increment(),
Addresstype::OpReturn => idxs.opreturnindex.copy_then_increment(),
Addresstype::PushOnly => idxs.pushonlyindex.copy_then_increment(),
Addresstype::Unknown => idxs.unknownindex.copy_then_increment(),
Addresstype::P2PK65 => idxs.p2pk65index.copy_then_increment(),
Addresstype::P2PK33 => idxs.p2pk33index.copy_then_increment(),
Addresstype::P2PKH => idxs.p2pkhindex.copy_then_increment(),
Addresstype::P2SH => idxs.p2shindex.copy_then_increment(),
Addresstype::P2WPKH => idxs.p2wpkhindex.copy_then_increment(),
Addresstype::P2WSH => idxs.p2wshindex.copy_then_increment(),
Addresstype::P2TR => idxs.p2trindex.copy_then_increment(),
};
vecs.addressindex_to_addresstype
@@ -537,13 +486,13 @@ impl Indexer<CACHED_GETS> {
let vout = Vout::from(outpoint.vout);
let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.get(&TxidPrefix::try_from(&txid)?)
.get(&TxidPrefix::from(&txid))
.context("txid should be in same block").inspect_err(|_| {
dbg!(&txid_prefix_to_txid_and_block_txindex_and_prev_txindex);
// panic!();
})?
.2;
let prev_txindex = txindex_global + block_txindex;
let prev_txindex = idxs.txindex + block_txindex;
let prev_txoutindex = new_txindexvout_to_txoutindex
.remove(&(prev_txindex, vout))
@@ -577,7 +526,7 @@ impl Indexer<CACHED_GETS> {
.into_iter()
.try_for_each(
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
let txindex = txindex_global + index;
let txindex = idxs.txindex + index;
txindex_to_tx_and_txid.insert(txindex, (tx, txid));
@@ -640,12 +589,19 @@ impl Indexer<CACHED_GETS> {
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
vecs.txindex_to_height.push_if_needed(txindex, height)?;
vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time.into())?;
// tx.base_size()
// tx.total_size()
// tx.is_explicitly_rbf()
// tx.weight()
// tx.vsize() in computer as it can be computed from the weight
Ok(())
})?;
txindex_global += Txindex::from(tx_len);
txinindex_global += Txinindex::from(inputs_len);
txoutindex_global += Txoutindex::from(outputs_len);
idxs.txindex += Txindex::from(tx_len);
idxs.txinindex += Txinindex::from(inputs_len);
idxs.txoutindex += Txoutindex::from(outputs_len);
idxs.push_future_if_needed(vecs)?;
let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked();
if should_snapshot {
@@ -655,7 +611,7 @@ impl Indexer<CACHED_GETS> {
Ok(())
})?;
export(trees, vecs, height)?;
export(trees, vecs, idxs.height)?;
sleep(Duration::from_millis(100));

View File

@@ -1,4 +1,8 @@
use std::{collections::BTreeMap, error, mem, path::Path};
use std::{
collections::{BTreeMap, BTreeSet},
error, mem,
path::Path,
};
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
@@ -17,8 +21,11 @@ pub struct Store<Key, Value> {
part: TransactionalPartitionHandle,
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
}
const CHECK_COLLISISONS: bool = true;
impl<K, V> Store<K, V>
where
K: Into<Slice> + Ord + Immutable + IntoBytes,
@@ -48,6 +55,7 @@ where
part,
rtx,
puts: BTreeMap::new(),
dels: BTreeSet::new(),
})
}
@@ -63,21 +71,44 @@ where
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
if !self.dels.is_empty() {
unreachable!("Shouldn't reach this");
// self.dels.remove(&key);
}
self.puts.insert(key, value);
}
}
pub fn remove(&mut self, key: K) {
if !self.puts.is_empty() {
unreachable!("Shouldn't reach this");
// self.puts.remove(&key);
}
self.dels.insert(key);
}
pub fn commit(&mut self, height: Height) -> Result<()> {
if self.has(height) && self.puts.is_empty() {
if self.has(height) && self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
}
self.meta.export(self.len(), height)?;
let mut wtx = self.keyspace.write_tx();
mem::take(&mut self.puts)
mem::take(&mut self.dels)
.into_iter()
.for_each(|(key, value)| wtx.insert(&self.part, key, value));
.for_each(|key| wtx.remove(&self.part, key));
mem::take(&mut self.puts).into_iter().for_each(|(key, value)| {
if CHECK_COLLISISONS {
if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) {
dbg!(value);
unreachable!();
}
}
wtx.insert(&self.part, key, value)
});
wtx.commit()?;
@@ -88,12 +119,12 @@ where
Ok(())
}
pub fn height(&self) -> Option<&Height> {
pub fn height(&self) -> Option<Height> {
self.meta.height()
}
pub fn len(&self) -> usize {
self.meta.len() + self.puts.len()
self.meta.len() + self.puts.len() - self.dels.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0

View File

@@ -67,8 +67,8 @@ impl StoreMeta {
path.join("version")
}
pub fn height(&self) -> Option<&Height> {
self.height.as_ref()
pub fn height(&self) -> Option<Height> {
self.height
}
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)

View File

@@ -1,16 +1,19 @@
use std::{path::Path, thread};
use storable_vec::Version;
use storable_vec::{Value, Version, CACHED_GETS};
use crate::{AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
use crate::{
AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, Indexes, TxidPrefix, Txindex,
};
mod base;
mod meta;
// mod version;
pub use base::*;
pub use meta::*;
use super::StorableVecs;
pub struct Fjalls {
pub addresshash_to_addressindex: Store<AddressHash, Addressindex>,
pub blockhash_prefix_to_height: Store<BlockHashPrefix, Height>,
@@ -30,135 +33,135 @@ impl Fjalls {
})
}
// pub fn rollback_from(
// &mut self,
// _wtx: &mut WriteTransaction,
// _height: Height,
// _exit: &Exit,
// ) -> color_eyre::Result<()> {
// panic!();
// let mut txindex = None;
pub fn rollback(&mut self, vecs: &StorableVecs<CACHED_GETS>, starting_indexes: &Indexes) -> color_eyre::Result<()> {
vecs.height_to_blockhash
.iter_from(starting_indexes.height, |(_, blockhash)| {
let blockhash = blockhash.as_ref();
let blockhash_prefix = BlockHashPrefix::from(blockhash);
self.blockhash_prefix_to_height.remove(blockhash_prefix);
Ok(())
})?;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
vecs.txindex_to_txid.iter_from(starting_indexes.txindex, |(_, txid)| {
let txid = txid.as_ref();
let txid_prefix = TxidPrefix::from(txid);
self.txid_prefix_to_txindex.remove(txid_prefix);
Ok(())
})?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
vecs.height_to_first_p2pk65index
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65addressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK65));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
vecs.height_to_first_p2pk33index
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33addressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK33));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
vecs.height_to_first_p2pkhindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PKH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// Ok(())
// })?;
vecs.height_to_first_p2shindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2shindex_to_p2shaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2SH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// let txindex = txindex.context("txindex to not be none by now")?;
vecs.height_to_first_p2trindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2trindex_to_p2traddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2TR));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
vecs.height_to_first_p2wpkhindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WPKH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
vecs.height_to_first_p2wshindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2wshindex_to_p2wshaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WSH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
// Ok(())
// })?;
self.commit(starting_indexes.height.decremented())?;
// let txoutindex = Txoutindex::from(txindex);
Ok(())
}
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// todo!("clear txindexvout_to_txoutindex")
// Ok(())
// }
pub fn min_height(&self) -> Option<Height> {
pub fn starting_height(&self) -> Height {
[
self.addresshash_to_addressindex.height(),
self.blockhash_prefix_to_height.height(),
self.txid_prefix_to_txindex.height(),
]
.into_iter()
.map(|height| height.map(Height::incremented).unwrap_or_default())
.min()
.flatten()
.cloned()
.unwrap()
}
pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
@@ -176,22 +179,4 @@ impl Fjalls {
Ok(())
})
}
// pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) {
// self.addressbytes_prefix_to_addressindex.update_meta(wtx, height);
// self.blockhash_prefix_to_height.update_meta(wtx, height);
// self.txid_prefix_to_txindex.update_meta(wtx, height);
// }
// pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
// thread::scope(|scope| {
// vec![
// scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
// scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
// scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
// ]
// .into_iter()
// .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
// })
// }
}

View File

@@ -28,14 +28,14 @@ where
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
if self.needs(height) {
height.write(&self.path_height())?;
}
height.write(&self.path_height())?;
self.vec.flush()
}
pub fn truncate_if_needed(&mut self, index: I, height: Height) -> storable_vec::Result<Option<T>> {
height.write(&self.path_height())?;
if self.height.is_none_or(|self_height| self_height != height) {
height.write(&self.path_height())?;
}
self.vec.truncate_if_needed(index)
}

View File

@@ -1,14 +1,17 @@
use std::{collections::BTreeMap, fs, io, path::Path};
use std::{fs, io, path::Path};
use exit::Exit;
use rayon::prelude::*;
use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS};
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex,
Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex,
P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes,
P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
use crate::{
structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime,
Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index,
P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes,
P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex,
Txinindex, Txoutindex, Unknownindex, Weight,
},
Indexes,
};
mod base;
@@ -58,8 +61,6 @@ pub struct StorableVecs<const MODE: u8> {
pub txoutindex_to_value: StorableVec<Txoutindex, Sats, MODE>,
}
// const UNSAFE_BLOCKS: usize = 1000;
impl<const MODE: u8> StorableVecs<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -180,113 +181,104 @@ impl<const MODE: u8> StorableVecs<MODE> {
})
}
#[allow(unused)]
pub fn rollback_from(&mut self, height: Height, exit: &Exit) -> storable_vec::Result<()> {
let prev_height = height.decremented();
pub fn rollback(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> {
let saved_height = starting_indexes.height.decremented();
let mut truncated_indexes: BTreeMap<String, Option<usize>> = BTreeMap::new();
// We don't want to override the starting indexes so we cut from n + 1
let height = starting_indexes.height.incremented();
let addressindex = self
.height_to_first_addressindex
.truncate_if_needed(height, prev_height)?;
let txindex = self.height_to_first_txindex.truncate_if_needed(height, prev_height)?;
let txinindex = self.height_to_first_txinindex.truncate_if_needed(height, prev_height)?;
let txoutindex = self
.height_to_first_txoutindex
.truncate_if_needed(height, prev_height)?;
let p2pk33index = self
.height_to_first_p2pk33index
.truncate_if_needed(height, prev_height)?;
let p2pk65index = self
.height_to_first_p2pk65index
.truncate_if_needed(height, prev_height)?;
let p2pkhindex = self
.height_to_first_p2pkhindex
.truncate_if_needed(height, prev_height)?;
let p2shindex = self.height_to_first_p2shindex.truncate_if_needed(height, prev_height)?;
let p2trindex = self.height_to_first_p2trindex.truncate_if_needed(height, prev_height)?;
let p2wpkhindex = self
.height_to_first_p2wpkhindex
.truncate_if_needed(height, prev_height)?;
let p2wshindex = self
.height_to_first_p2wshindex
.truncate_if_needed(height, prev_height)?;
self.height_to_blockhash.truncate_if_needed(height, prev_height)?;
self.height_to_difficulty.truncate_if_needed(height, prev_height)?;
self.height_to_first_addressindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_emptyindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_multisigindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_opreturnindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2pk33index
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2pk65index
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2pkhindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2shindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2trindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2wpkhindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_p2wshindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_pushonlyindex
.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txindex.truncate_if_needed(height, saved_height)?;
self.height_to_first_txinindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txoutindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_unknownindex
.truncate_if_needed(height, prev_height)?;
self.height_to_size.truncate_if_needed(height, prev_height)?;
self.height_to_timestamp.truncate_if_needed(height, prev_height)?;
self.height_to_weight.truncate_if_needed(height, prev_height)?;
.truncate_if_needed(height, saved_height)?;
if let Some(addressindex) = addressindex {
self.addressindex_to_addresstype
.truncate_if_needed(addressindex, prev_height)?;
self.addressindex_to_addresstypeindex
.truncate_if_needed(addressindex, prev_height)?;
self.addressindex_to_height
.truncate_if_needed(addressindex, prev_height)?;
}
// Now we can cut everything that's out of date
let &Indexes {
addressindex,
height,
p2pk33index,
p2pk65index,
p2pkhindex,
p2shindex,
p2trindex,
p2wpkhindex,
p2wshindex,
txindex,
txinindex,
txoutindex,
..
} = starting_indexes;
if let Some(p2pk33index) = p2pk33index {
self.p2pk33index_to_p2pk33addressbytes
.truncate_if_needed(p2pk33index, prev_height)?;
}
if let Some(p2pk65index) = p2pk65index {
self.p2pk65index_to_p2pk65addressbytes
.truncate_if_needed(p2pk65index, prev_height)?;
}
if let Some(p2pkhindex) = p2pkhindex {
self.p2pkhindex_to_p2pkhaddressbytes
.truncate_if_needed(p2pkhindex, prev_height)?;
}
if let Some(p2shindex) = p2shindex {
self.p2shindex_to_p2shaddressbytes
.truncate_if_needed(p2shindex, prev_height)?;
}
if let Some(p2trindex) = p2trindex {
self.p2trindex_to_p2traddressbytes
.truncate_if_needed(p2trindex, prev_height)?;
}
if let Some(p2wpkhindex) = p2wpkhindex {
self.p2wpkhindex_to_p2wpkhaddressbytes
.truncate_if_needed(p2wpkhindex, prev_height)?;
}
if let Some(p2wshindex) = p2wshindex {
self.p2wshindex_to_p2wshaddressbytes
.truncate_if_needed(p2wshindex, prev_height);
}
self.height_to_blockhash.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty.truncate_if_needed(height, saved_height)?;
self.height_to_size.truncate_if_needed(height, saved_height)?;
self.height_to_timestamp.truncate_if_needed(height, saved_height)?;
self.height_to_weight.truncate_if_needed(height, saved_height)?;
if let Some(txindex) = txindex {
self.txindex_to_first_txinindex
.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_first_txoutindex
.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_height.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_locktime.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_txid.truncate_if_needed(txindex, prev_height)?;
self.txindex_to_txversion.truncate_if_needed(txindex, prev_height)?;
}
self.addressindex_to_addresstype
.truncate_if_needed(addressindex, saved_height)?;
self.addressindex_to_addresstypeindex
.truncate_if_needed(addressindex, saved_height)?;
self.addressindex_to_height
.truncate_if_needed(addressindex, saved_height)?;
if let Some(txinindex) = txinindex {
self.txinindex_to_txoutindex
.truncate_if_needed(txinindex, prev_height)?;
}
self.p2pk33index_to_p2pk33addressbytes
.truncate_if_needed(p2pk33index, saved_height)?;
self.p2pk65index_to_p2pk65addressbytes
.truncate_if_needed(p2pk65index, saved_height)?;
self.p2pkhindex_to_p2pkhaddressbytes
.truncate_if_needed(p2pkhindex, saved_height)?;
self.p2shindex_to_p2shaddressbytes
.truncate_if_needed(p2shindex, saved_height)?;
self.p2trindex_to_p2traddressbytes
.truncate_if_needed(p2trindex, saved_height)?;
self.p2wpkhindex_to_p2wpkhaddressbytes
.truncate_if_needed(p2wpkhindex, saved_height)?;
self.p2wshindex_to_p2wshaddressbytes
.truncate_if_needed(p2wshindex, saved_height)?;
if let Some(txoutindex) = txoutindex {
self.txoutindex_to_addressindex
.truncate_if_needed(txoutindex, prev_height)?;
self.txoutindex_to_value.truncate_if_needed(txoutindex, prev_height)?;
}
self.txindex_to_first_txinindex
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_first_txoutindex
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_height.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_locktime.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txid.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txversion.truncate_if_needed(txindex, saved_height)?;
self.txinindex_to_txoutindex
.truncate_if_needed(txinindex, saved_height)?;
self.txoutindex_to_addressindex
.truncate_if_needed(txoutindex, saved_height)?;
self.txoutindex_to_value.truncate_if_needed(txoutindex, saved_height)?;
Ok(())
}
@@ -297,12 +289,12 @@ impl<const MODE: u8> StorableVecs<MODE> {
.try_for_each(|vec| vec.flush(height))
}
pub fn min_height(&mut self) -> color_eyre::Result<Option<Height>> {
Ok(self
.as_mut_any_vec_slice()
pub fn starting_height(&mut self) -> Height {
self.as_mut_any_vec_slice()
.into_iter()
.map(|vec| vec.height().unwrap_or_default())
.min())
.map(|vec| vec.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 40] {

View File

@@ -1,9 +1,12 @@
use std::mem;
use derive_deref::Deref;
use iterator::rpc::{Client, RpcApi};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct BlockHash([u8; 32]);
@@ -18,3 +21,10 @@ impl From<BlockHash> for bitcoin::BlockHash {
unsafe { mem::transmute(value) }
}
}
impl TryFrom<(&Client, Height)> for BlockHash {
type Error = iterator::rpc::Error;
fn try_from((rpc, height): (&Client, Height)) -> Result<Self, Self::Error> {
Ok(Self::from(rpc.get_block_hash(u64::from(height))?))
}
}

View File

@@ -41,10 +41,9 @@ impl From<AddressHash> for Slice {
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct BlockHashPrefix([u8; 8]);
impl TryFrom<&BlockHash> for BlockHashPrefix {
type Error = color_eyre::Report;
fn try_from(value: &BlockHash) -> Result<Self, Self::Error> {
Ok(Self(copy_first_8bytes(&value[..])))
impl From<&BlockHash> for BlockHashPrefix {
fn from(value: &BlockHash) -> Self {
Self(copy_first_8bytes(&value[..]).unwrap())
}
}
impl TryFrom<Slice> for BlockHashPrefix {
@@ -66,10 +65,9 @@ impl From<BlockHashPrefix> for Slice {
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct TxidPrefix([u8; 8]);
impl TryFrom<&Txid> for TxidPrefix {
type Error = color_eyre::Report;
fn try_from(value: &Txid) -> Result<Self, Self::Error> {
Ok(Self(copy_first_8bytes(&value[..])))
impl From<&Txid> for TxidPrefix {
fn from(value: &Txid) -> Self {
Self(copy_first_8bytes(&value[..]).unwrap())
}
}
impl TryFrom<Slice> for TxidPrefix {
@@ -89,14 +87,14 @@ impl From<TxidPrefix> for Slice {
}
}
fn copy_first_8bytes(slice: &[u8]) -> [u8; 8] {
fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8], ()> {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
panic!("bad len");
return Err(());
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
buf
Ok(buf)
}

View File

@@ -30,13 +30,31 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
pub struct Height(u32);
impl Height {
const ZERO: Self = Height(0);
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.as_bytes())
}
pub fn decremented(&self) -> Self {
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(self.0 + 1)
}
pub fn decrement(&mut self) {
self.0 -= 1;
}
pub fn decremented(self) -> Self {
Self(self.0.checked_sub(1).unwrap_or_default())
}
pub fn is_zero(self) -> bool {
self == Self::ZERO
}
}
impl PartialEq<u64> for Height {
@@ -133,6 +151,12 @@ impl From<Height> for usize {
}
}
impl From<Height> for u64 {
fn from(value: Height) -> Self {
value.0 as u64
}
}
impl TryFrom<&Path> for Height {
type Error = storable_vec::Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {

View File

@@ -0,0 +1,114 @@
use color_eyre::eyre::ContextCompat;
use iterator::rpc::Client;
use iterator::NUMBER_OF_UNSAFE_BLOCKS;
use storable_vec::CACHED_GETS;
use crate::storage::{Fjalls, StorableVecs};
use super::{
Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex,
P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
};
#[derive(Debug, Default)]
pub struct Indexes {
pub addressindex: Addressindex,
pub emptyindex: Emptyindex,
pub height: Height,
pub multisigindex: Multisigindex,
pub opreturnindex: Opreturnindex,
pub p2pk33index: P2PK33index,
pub p2pk65index: P2PK65index,
pub p2pkhindex: P2PKHindex,
pub p2shindex: P2SHindex,
pub p2trindex: P2TRindex,
pub p2wpkhindex: P2WPKHindex,
pub p2wshindex: P2WSHindex,
pub pushonlyindex: Pushonlyindex,
pub txindex: Txindex,
pub txinindex: Txinindex,
pub txoutindex: Txoutindex,
pub unknownindex: Unknownindex,
}
impl Indexes {
pub fn push_if_needed(&self, vecs: &mut StorableVecs<CACHED_GETS>) -> storable_vec::Result<()> {
let height = self.height;
vecs.height_to_first_txindex.push_if_needed(height, self.txindex)?;
vecs.height_to_first_txinindex.push_if_needed(height, self.txinindex)?;
vecs.height_to_first_txoutindex
.push_if_needed(height, self.txoutindex)?;
vecs.height_to_first_addressindex
.push_if_needed(height, self.addressindex)?;
vecs.height_to_first_emptyindex
.push_if_needed(height, self.emptyindex)?;
vecs.height_to_first_multisigindex
.push_if_needed(height, self.multisigindex)?;
vecs.height_to_first_opreturnindex
.push_if_needed(height, self.opreturnindex)?;
vecs.height_to_first_pushonlyindex
.push_if_needed(height, self.pushonlyindex)?;
vecs.height_to_first_unknownindex
.push_if_needed(height, self.unknownindex)?;
vecs.height_to_first_p2pk33index
.push_if_needed(height, self.p2pk33index)?;
vecs.height_to_first_p2pk65index
.push_if_needed(height, self.p2pk65index)?;
vecs.height_to_first_p2pkhindex
.push_if_needed(height, self.p2pkhindex)?;
vecs.height_to_first_p2shindex.push_if_needed(height, self.p2shindex)?;
vecs.height_to_first_p2trindex.push_if_needed(height, self.p2trindex)?;
vecs.height_to_first_p2wpkhindex
.push_if_needed(height, self.p2wpkhindex)?;
vecs.height_to_first_p2wshindex
.push_if_needed(height, self.p2wshindex)?;
Ok(())
}
pub fn push_future_if_needed(&mut self, vecs: &mut StorableVecs<CACHED_GETS>) -> storable_vec::Result<()> {
self.height.increment();
self.push_if_needed(vecs)?;
self.height.decrement();
Ok(())
}
}
impl TryFrom<(&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)> for Indexes {
type Error = color_eyre::Report;
fn try_from((vecs, trees, rpc): (&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)) -> color_eyre::Result<Self> {
// Height at which we wanna start: min last saved + 1 or 0
let starting_height = vecs.starting_height().min(trees.starting_height());
// But we also need to check the chain and start earlier in case of a reorg
let height = (starting_height
.checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32)
.unwrap_or_default()..*starting_height) // ..= because of last saved + 1
.map(Height::from)
.find(|height| {
let rpc_blockhash = BlockHash::try_from((rpc, *height)).unwrap();
let saved_blockhash = vecs.height_to_blockhash.get(*height).unwrap().unwrap();
&rpc_blockhash != saved_blockhash.as_ref()
})
.unwrap_or(starting_height);
Ok(Self {
addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?,
emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?,
height,
multisigindex: *vecs.height_to_first_multisigindex.get(height)?.context("")?,
opreturnindex: *vecs.height_to_first_opreturnindex.get(height)?.context("")?,
p2pk33index: *vecs.height_to_first_p2pk33index.get(height)?.context("")?,
p2pk65index: *vecs.height_to_first_p2pk65index.get(height)?.context("")?,
p2pkhindex: *vecs.height_to_first_p2pkhindex.get(height)?.context("")?,
p2shindex: *vecs.height_to_first_p2shindex.get(height)?.context("")?,
p2trindex: *vecs.height_to_first_p2trindex.get(height)?.context("")?,
p2wpkhindex: *vecs.height_to_first_p2wpkhindex.get(height)?.context("")?,
p2wshindex: *vecs.height_to_first_p2wshindex.get(height)?.context("")?,
pushonlyindex: *vecs.height_to_first_pushonlyindex.get(height)?.context("")?,
txindex: *vecs.height_to_first_txindex.get(height)?.context("")?,
txinindex: *vecs.height_to_first_txinindex.get(height)?.context("")?,
txoutindex: *vecs.height_to_first_txoutindex.get(height)?.context("")?,
unknownindex: *vecs.height_to_first_unknownindex.get(height)?.context("")?,
})
}
}

View File

@@ -5,6 +5,7 @@ mod addresstypeindex;
mod blockhash;
mod compressed;
mod height;
mod indexes;
mod locktime;
mod sats;
mod timestamp;
@@ -24,6 +25,7 @@ pub use addresstypeindex::*;
pub use blockhash::*;
pub use compressed::*;
pub use height::*;
pub use indexes::*;
pub use locktime::*;
pub use sats::*;
pub use timestamp::*;