Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-14 20:48:12 -07:00
Code Issues Packages Projects Releases Wiki Activity

workspace: reorg

Browse Source
This commit is contained in:
nym21
2025-01-28 17:45:36 +01:00
parent f7f3e3cc03
commit 8c610f8a83
66 changed files with 268 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
indexer/src/lib.rs
+679
View File
@@ -0,0 +1,679 @@
use std::{
collections::BTreeMap,
io::{Read, Write},
path::Path,
str::FromStr,
thread::{self, sleep},
time::Duration,
};
pub use biter::*;
use bitcoin::{Transaction, TxIn, TxOut, Txid};
use color_eyre::eyre::{eyre, ContextCompat};
use exit::Exit;
use rayon::prelude::*;
mod storage;
mod structs;
pub use biter;
use storage::{Fjalls, StorableVecs};
pub use structs::{
AddressHash, Addressbytes, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp, TxidPrefix,
Txindex, Txinindex, Txoutindex, Vin, Vout,
};
const UNSAFE_BLOCKS: u32 = 100;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
pub struct Indexer {
vecs: StorableVecs,
parts: Fjalls,
}
impl Indexer {
pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
Ok(Self {
vecs: StorableVecs::import(&indexes_dir.join("vecs"))?,
parts: Fjalls::import(&indexes_dir.join("fjall"))?,
})
}
pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
let check_collisions = true;
let vecs = &mut self.vecs;
let parts = &mut self.parts;
let mut height = vecs
.min_height()
.unwrap_or_default()
.min(parts.min_height())
.and_then(|h| h.checked_sub(UNSAFE_BLOCKS))
.map(Height::from)
.unwrap_or_default();
let mut txindex_global = vecs.height_to_first_txindex.get_or_default(height)?;
let mut txinindex_global = vecs.height_to_first_txinindex.get_or_default(height)?;
let mut txoutindex_global = vecs.height_to_first_txoutindex.get_or_default(height)?;
let mut addressindex_global = vecs.height_to_first_addressindex.get_or_default(height)?;
let mut emptyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut multisigindex_global = vecs.height_to_first_multisigindex.get_or_default(height)?;
let mut opreturnindex_global = vecs.height_to_first_opreturnindex.get_or_default(height)?;
let mut pushonlyindex_global = vecs.height_to_first_pushonlyindex.get_or_default(height)?;
let mut unknownindex_global = vecs.height_to_first_unknownindex.get_or_default(height)?;
let mut p2pk33index_global = vecs.height_to_first_p2pk33index.get_or_default(height)?;
let mut p2pk65index_global = vecs.height_to_first_p2pk65index.get_or_default(height)?;
let mut p2pkhindex_global = vecs.height_to_first_p2pkhindex.get_or_default(height)?;
let mut p2shindex_global = vecs.height_to_first_p2shindex.get_or_default(height)?;
let mut p2trindex_global = vecs.height_to_first_p2trindex.get_or_default(height)?;
let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
let export = |parts: &mut Fjalls, vecs: &mut StorableVecs, height: Height| -> color_eyre::Result<()> {
println!("Exporting...");
exit.block();
thread::scope(|scope| -> color_eyre::Result<()> {
let vecs_handle = scope.spawn(|| vecs.flush(height));
parts.commit(height)?;
vecs_handle.join().unwrap()?;
Ok(())
})?;
exit.unblock();
Ok(())
};
biter::new(bitcoin_dir, Some(height.into()), None, rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
height = Height::from(_height);
let timestamp = Timestamp::try_from(block.header.time)?;
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
if &blockhash != saved_blockhash.as_ref() {
todo!("Rollback not implemented");
// parts.rollback_from(&mut rtx, height, &exit)?;
}
}
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
if parts
.blockhash_prefix_to_height
.get(&blockhash_prefix)?
.is_some_and(|prev_height| *prev_height != height)
{
dbg!(blockhash);
return Err(eyre!("Collision, expect prefix to need be set yet"));
}
parts
.blockhash_prefix_to_height
.insert_if_needed(blockhash_prefix, height, height);
vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
vecs.height_to_timestamp.push_if_needed(height, timestamp)?;
vecs.height_to_size.push_if_needed(height, block.total_size())?;
vecs.height_to_weight.push_if_needed(height, block.weight())?;
vecs.height_to_first_txindex.push_if_needed(height, txindex_global)?;
vecs.height_to_first_txinindex
.push_if_needed(height, txinindex_global)?;
vecs.height_to_first_txoutindex
.push_if_needed(height, txoutindex_global)?;
vecs.height_to_first_addressindex
.push_if_needed(height, addressindex_global)?;
vecs.height_to_first_emptyindex
.push_if_needed(height, emptyindex_global)?;
vecs.height_to_first_multisigindex
.push_if_needed(height, multisigindex_global)?;
vecs.height_to_first_opreturnindex
.push_if_needed(height, opreturnindex_global)?;
vecs.height_to_first_pushonlyindex
.push_if_needed(height, pushonlyindex_global)?;
vecs.height_to_first_unknownindex
.push_if_needed(height, unknownindex_global)?;
vecs.height_to_first_p2pk33index.push_if_needed(height, p2pk33index_global)?;
vecs.height_to_first_p2pk65index.push_if_needed(height, p2pk65index_global)?;
vecs.height_to_first_p2pkhindex.push_if_needed(height, p2pkhindex_global)?;
vecs.height_to_first_p2shindex.push_if_needed(height, p2shindex_global)?;
vecs.height_to_first_p2trindex.push_if_needed(height, p2trindex_global)?;
vecs.height_to_first_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?;
vecs.height_to_first_p2wshindex.push_if_needed(height, p2wshindex_global)?;
let inputs = block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.input
.iter()
.enumerate()
.map(move |(vin, txin)| (Txindex::from(index), Vin::from(vin), txin, tx))
})
.collect::<Vec<_>>();
let outputs = block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (Txindex::from(index), Vout::from(vout), txout, tx))
})
.collect::<Vec<_>>();
let tx_len = block.txdata.len();
let outputs_len = outputs.len();
let inputs_len = inputs.len();
let (
txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle,
input_source_vec_handle,
txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle,
) = thread::scope(|scope| {
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle =
scope.spawn(|| -> color_eyre::Result<_> {
block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| -> color_eyre::Result<_> {
let txid = tx.compute_txid();
let txid_prefix = TxidPrefix::try_from(&txid)?;
let prev_txindex_opt =
if check_collisions && parts.txid_prefix_to_txindex.needs(height) {
// Should only find collisions for two txids (duplicates), see below
parts.txid_prefix_to_txindex.get(&txid_prefix)?.map(|v| *v)
} else {
None
};
Ok((txid_prefix, (tx, txid, Txindex::from(index), prev_txindex_opt)))
})
.try_fold(BTreeMap::new, |mut map, tuple| {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
})
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
}
})
});
let input_source_vec_handle = scope.spawn(|| {
inputs
.into_par_iter()
.enumerate()
.map(|(block_txinindex, (block_txindex, vin, txin, tx))| -> color_eyre::Result<(Txinindex, InputSource)> {
let txindex = txindex_global + block_txindex;
let txinindex = txinindex_global + Txinindex::from(block_txinindex);
// dbg!((txindex, txinindex, vin));
let outpoint = txin.previous_output;
if tx.is_coinbase() {
return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
}
let prev_txindex = if let Some(txindex) = parts
.txid_prefix_to_txindex
.get(&TxidPrefix::try_from(&outpoint.txid)?)?
.map(|v| *v)
.and_then(|txindex| {
// Checking if not finding txindex from the future
(txindex < txindex_global).then_some(txindex)
}) {
txindex
} else {
// dbg!(txindex_global + block_txindex, txindex, txin, vin);
return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
};
let vout = Vout::from(outpoint.vout);
let txoutindex = *vecs
.txindex_to_first_txoutindex
.get(prev_txindex)?
.context("Expect txoutindex to not be none")
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
})?
+ vout;
Ok((txinindex, InputSource::PreviousBlock((
vin,
txindex,
txoutindex,
))))
})
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
})
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
}
})
});
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| {
outputs
.into_par_iter()
.enumerate()
.map(
#[allow(clippy::type_complexity)]
|(block_txoutindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<(
Txoutindex,
(
&TxOut,
Txindex,
Vout,
Addresstype,
color_eyre::Result<Addressbytes>,
Option<Addressindex>,
&Transaction,
),
)> {
let txindex = txindex_global + block_txindex;
let txoutindex = txoutindex_global + Txoutindex::from(block_txoutindex);
let script = &txout.script_pubkey;
let addresstype = Addresstype::from(script);
let addressbytes_res =
Addressbytes::try_from((script, addresstype)).inspect_err(|_| {
// dbg!(&txout, height, txi, &tx.compute_txid());
});
let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
parts
.addresshash_to_addressindex
.get(&AddressHash::from((addressbytes, addresstype)))
.unwrap()
.map(|v| *v)
// Checking if not in the future
.and_then(|addressindex_local| {
(addressindex_local < addressindex_global).then_some(addressindex_local)
})
}); // OK
if let Some(Some(addressindex)) = check_collisions.then_some(addressindex_opt) {
let addressbytes = addressbytes_res.as_ref().unwrap();
let prev_addresstype = *vecs
.addressindex_to_addresstype
.get(addressindex)?
.context("Expect to have address type")?;
let addresstypeindex = *vecs
.addressindex_to_addresstypeindex
.get(addressindex)?
.context("Expect to have address type index")?;
// Good first time
// Wrong after rerun
let prev_addressbytes_opt =
vecs.get_addressbytes(prev_addresstype, addresstypeindex)?;
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
if (vecs.addressindex_to_addresstype.hasnt(addressindex)
&& addresstype != prev_addresstype)
|| (parts.addresshash_to_addressindex.needs(height)
&& prev_addressbytes != addressbytes)
{
let txid = tx.compute_txid();
dbg!(
_height,
txid,
vout,
block_txindex,
addresstype,
prev_addresstype,
prev_addressbytes,
addressbytes,
addressindex_global,
addressindex,
addresstypeindex,
txout,
AddressHash::from((addressbytes, addresstype)),
AddressHash::from((prev_addressbytes, prev_addresstype))
);
panic!()
}
}
Ok((
txoutindex,
(
txout,
txindex,
vout,
addresstype,
addressbytes_res,
addressindex_opt,
tx,
),
))
},
)
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
})
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
}
})
});
(
txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(),
input_source_vec_handle.join(),
txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.join(),
)
});
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex =
txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle
.ok()
.context(
"Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join",
)??;
let input_source_vec = input_source_vec_handle
.ok()
.context("Export input_source_vec_handle to join")??;
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt =
txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle
.ok()
.context(
"Expect txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle to join",
)??;
let mut new_txindexvout_to_txoutindex: BTreeMap<
(Txindex, Vout),
Txoutindex,
> = BTreeMap::new();
let mut already_added_addresshash: BTreeMap<AddressHash, Addressindex> = BTreeMap::new();
txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt
.into_iter()
.try_for_each(
|(
txoutindex,
(txout, txindex, vout, addresstype, addressbytes_res, addressindex_opt, _tx),
)|
-> color_eyre::Result<()> {
let amount = Amount::from(txout.value);
if vout.is_zero() {
vecs.txindex_to_first_txoutindex.push_if_needed(txindex, txoutindex)?;
}
vecs.txoutindex_to_amount.push_if_needed(txoutindex, amount)?;
let mut addressindex = addressindex_global;
let mut addresshash = None;
if let Some(addressindex_local) = addressindex_opt.or_else(|| {
addressbytes_res.as_ref().ok().and_then(|addressbytes| {
// Check if address was first seen before in this iterator
// Example: https://mempool.space/address/046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0c
addresshash.replace(AddressHash::from((addressbytes, addresstype)));
already_added_addresshash
.get(addresshash.as_ref().unwrap())
.cloned()
})
}) {
addressindex = addressindex_local;
} else {
addressindex_global.increment();
let addresstypeindex = match addresstype {
Addresstype::Empty => emptyindex_global.clone_then_increment(),
Addresstype::Multisig => multisigindex_global.clone_then_increment(),
Addresstype::OpReturn => opreturnindex_global.clone_then_increment(),
Addresstype::PushOnly => pushonlyindex_global.clone_then_increment(),
Addresstype::Unknown => unknownindex_global.clone_then_increment(),
Addresstype::P2PK65 => p2pk65index_global.clone_then_increment(),
Addresstype::P2PK33 => p2pk33index_global.clone_then_increment(),
Addresstype::P2PKH => p2pkhindex_global.clone_then_increment(),
Addresstype::P2SH => p2shindex_global.clone_then_increment(),
Addresstype::P2WPKH => p2wpkhindex_global.clone_then_increment(),
Addresstype::P2WSH => p2wshindex_global.clone_then_increment(),
Addresstype::P2TR => p2trindex_global.clone_then_increment(),
};
vecs.addressindex_to_addresstype
.push_if_needed(addressindex, addresstype)?;
vecs.addressindex_to_addresstypeindex
.push_if_needed(addressindex, addresstypeindex)?;
vecs.addressindex_to_height
.push_if_needed(addressindex, height)?;
if let Ok(addressbytes) = addressbytes_res {
let addresshash = addresshash.unwrap();
already_added_addresshash
.insert(addresshash, addressindex);
parts.addresshash_to_addressindex.insert_if_needed(
addresshash,
addressindex,
height,
);
vecs.push_addressbytes_if_needed(addresstypeindex, addressbytes)?;
}
}
new_txindexvout_to_txoutindex
.insert((txindex, vout), txoutindex);
vecs.txoutindex_to_addressindex
.push_if_needed(txoutindex, addressindex)?;
Ok(())
},
)?;
drop(already_added_addresshash);
input_source_vec
.into_iter()
.map(
#[allow(clippy::type_complexity)]
|(txinindex, input_source)| -> color_eyre::Result<(
Txinindex, Vin, Txindex, Txoutindex
)> {
match input_source {
InputSource::PreviousBlock((vin, txindex, txoutindex)) => Ok((txinindex, vin, txindex, txoutindex)),
InputSource::SameBlock((tx, txindex, txin, vin)) => {
if tx.is_coinbase() {
return Ok((txinindex, vin, txindex, Txoutindex::COINBASE));
}
let outpoint = txin.previous_output;
let txid = outpoint.txid;
let vout = Vout::from(outpoint.vout);
let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.get(&TxidPrefix::try_from(&txid)?)
.context("txid should be in same block").inspect_err(|_| {
dbg!(&txid_prefix_to_txid_and_block_txindex_and_prev_txindex);
// panic!();
})?
.2;
let prev_txindex = txindex_global + block_txindex;
let prev_txoutindex = new_txindexvout_to_txoutindex
.remove(&(prev_txindex, vout))
.context("should have found addressindex from same block")
.inspect_err(|_| {
dbg!(&new_txindexvout_to_txoutindex, txin, prev_txindex, vout, txid);
})?;
Ok((txinindex, vin, txindex, prev_txoutindex))
}
}
},
)
.try_for_each(|res| -> color_eyre::Result<()> {
let (txinindex, vin, txindex, txoutindex) = res?;
if vin.is_zero() {
vecs.txindex_to_first_txinindex.push_if_needed(txindex, txinindex)?;
}
vecs.txinindex_to_txoutindex.push_if_needed(txinindex, txoutindex)?;
Ok(())
})?;
drop(new_txindexvout_to_txoutindex);
let mut txindex_to_tx_and_txid: BTreeMap<Txindex, (&Transaction, Txid)> = BTreeMap::default();
txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.into_iter()
.try_for_each(
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
let txindex = txindex_global + index;
txindex_to_tx_and_txid.insert(txindex, (tx, txid));
match prev_txindex_opt {
None => {
parts
.txid_prefix_to_txindex
.insert_if_needed(txid_prefix, txindex, height);
}
Some(prev_txindex) => {
// In case if we start at an already parsed height
if txindex == prev_txindex {
return Ok(());
}
let len = vecs.txindex_to_txid.len();
// Ok if `get` is not par as should happen only twice
let prev_txid = vecs
.txindex_to_txid
.get(prev_txindex)?
.context("To have txid for txindex")
.inspect_err(|_| {
dbg!(txindex, txid, len);
})?;
// #[allow(clippy::redundant_locals)]
// let prev_txid = prev_txid;
let prev_txid = prev_txid.as_ref();
// If another Txid needs to be added to the list
// We need to check that it's also a coinbase tx otherwise par_iter inputs needs to be updated
let only_known_dup_txids = [
Txid::from_str(
"d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599",
)?,
Txid::from_str(
"e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468",
)?,
];
let is_dup = only_known_dup_txids.contains(prev_txid);
if !is_dup {
let prev_height =
vecs.txindex_to_height.get(prev_txindex)?.expect("To have height");
dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
}
}
}
Ok(())
},
)?;
txindex_to_tx_and_txid
.into_iter()
.try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
vecs.txindex_to_txversion.push_if_needed(txindex, tx.version)?;
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
vecs.txindex_to_height.push_if_needed(txindex, height)?;
vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time)?;
Ok(())
})?;
txindex_global += Txindex::from(tx_len);
txinindex_global += Txinindex::from(inputs_len);
txoutindex_global += Txoutindex::from(outputs_len);
let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active();
if should_snapshot {
export(parts, vecs, height)?;
}
Ok(())
})?;
export(parts, vecs, height)?;
sleep(Duration::from_millis(100));
Ok(())
}
}
#[derive(Debug)]
enum InputSource<'a> {
PreviousBlock((Vin, Txindex, Txoutindex)),
SameBlock((&'a Transaction, Txindex, &'a TxIn, Vin)),
}
#[allow(unused)]
fn pause() {
let mut stdin = std::io::stdin();
let mut stdout = std::io::stdout();
write!(stdout, "Press any key to continue...").unwrap();
stdout.flush().unwrap();
let _ = stdin.read(&mut [0u8]).unwrap();
}
indexer/src/main.rs
+26
View File
@@ -0,0 +1,26 @@
use std::path::Path;
use bindex::Indexer;
use biter::rpc;
use exit::Exit;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let data_dir = Path::new("../../bitcoin");
let rpc = rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
)?;
let exit = Exit::new();
let i = std::time::Instant::now();
let mut indexer = Indexer::import(Path::new("indexes"))?;
indexer.index(data_dir, rpc, &exit)?;
dbg!(i.elapsed());
Ok(())
}
indexer/src/storage/canopies/database.rs
+50
View File
@@ -0,0 +1,50 @@
use std::{
ops::{Deref, DerefMut},
time::Duration,
};
use canopydb::{Database as CanopyDatabase, DbOptions, Error, WriteTransaction};
use super::Environment;
#[derive(Debug)]
pub struct Database {
db: CanopyDatabase,
// pub wtx: WriteTransaction,
}
impl Deref for Database {
type Target = CanopyDatabase;
fn deref(&self) -> &Self::Target {
&self.db
}
}
impl DerefMut for Database {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.db
}
}
impl Database {
pub fn new(environment: &Environment, name: &str) -> color_eyre::Result<Self> {
let mut options = DbOptions::default();
options.use_wal = false;
options.checkpoint_interval = Duration::from_secs(u64::MAX);
options.checkpoint_target_size = usize::MAX;
options.throttle_memory_limit = usize::MAX;
options.stall_memory_limit = usize::MAX;
options.write_txn_memory_limit = usize::MAX;
let db = environment.get_or_create_database_with(name, options)?;
Ok(Self {
// wtx: db.begin_write()?,
db,
})
}
pub fn flush(&mut self) -> Result<(), Error> {
// drop(blockhash_prefix_to_height_tree);
// blockhash_prefix_to_height_tx_opt.take().map(|tx| tx.commit());
self.checkpoint()
}
}
indexer/src/storage/canopies/environment.rs
+20
View File
@@ -0,0 +1,20 @@
use std::path::Path;
use canopydb::{EnvOptions, Environment as CanopyEnvironment};
use derive_deref::{Deref, DerefMut};
#[derive(Debug, Deref, DerefMut)]
pub struct Environment(CanopyEnvironment);
impl Environment {
pub fn new(path: &Path) -> color_eyre::Result<Self> {
let mut options = EnvOptions::new(path);
// options.use_mmap = true;
options.disable_fsync = true;
options.wal_new_file_on_checkpoint = false;
options.wal_background_sync_interval = None;
options.wal_write_batch_memory_limit = usize::MAX;
Ok(Self(CanopyEnvironment::with_options(options)?))
}
}
indexer/src/storage/canopies/mod.rs
+9
View File
@@ -0,0 +1,9 @@
mod database;
mod environment;
// mod transaction;
mod tree;
pub use database::*;
pub use environment::*;
// pub use transaction::*;
pub use tree::*;
indexer/src/storage/canopies/transaction.rs
+19
View File
@@ -0,0 +1,19 @@
use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
use super::{Database, Tree};
#[derive(Debug)]
pub struct Transaction<'a, K, V> {
tx: WriteTransaction,
tree: Tree<'a, K, V>,
}
impl<'a, K, V> Transaction<'a, K, V> {
pub fn new(db: &Database) -> color_eyre::Result<Self> {
let tx = db.begin_write()?;
let tree = Tree::new(&tx)?;
Ok(Self { tx, tree })
}
}
indexer/src/storage/canopies/tree.rs
+84
View File
@@ -0,0 +1,84 @@
use std::{
fmt::Debug,
marker::PhantomData,
ops::{Deref, DerefMut},
};
use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
use color_eyre::eyre::eyre;
#[derive(Debug)]
pub struct Tree<'a, K, V> {
tree: CanopyTree<'a>,
k: PhantomData<K>,
v: PhantomData<V>,
}
impl<'a, K, V> Deref for Tree<'a, K, V> {
type Target = CanopyTree<'a>;
fn deref(&self) -> &Self::Target {
&self.tree
}
}
impl<'a, K, V> DerefMut for Tree<'a, K, V> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.tree
}
}
impl<'a, K, V> Tree<'a, K, V>
where
K: Debug + Sized,
V: Debug + Sized + Clone + Copy,
{
const SIZE_OF_K: usize = size_of::<K>();
const SIZE_OF_V: usize = size_of::<V>();
pub fn new(tx: &'a WriteTransaction) -> color_eyre::Result<Self> {
let mut options = TreeOptions::new();
options.compress_overflow_values = None;
options.fixed_key_len = size_of::<K>() as i8;
options.fixed_value_len = size_of::<V>() as i8;
Ok(Self {
tree: tx.get_or_create_tree_with(b"tree", options)?,
k: PhantomData,
v: PhantomData,
})
}
pub fn get(&self, key: &K) -> color_eyre::Result<Option<V>> {
let slice = self.tree.get(Self::key_as_slice(key))?;
if slice.is_none() {
return Ok(None);
}
let slice = slice.unwrap();
let (prefix, shorts, suffix) = unsafe { slice.align_to::<V>() };
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
dbg!(&key, &prefix, &shorts, &suffix);
return Err(eyre!("align_to issue"));
}
Ok(Some(shorts[0]))
}
pub fn insert(&mut self, key: &K, value: &V) -> Result<(), canopydb::Error> {
self.tree
.insert(Self::key_as_slice(key), Self::value_as_slice(value))
}
fn key_as_slice(key: &K) -> &[u8] {
let data: *const K = key;
let data: *const u8 = data as *const u8;
unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_K) }
}
fn value_as_slice(value: &V) -> &[u8] {
let data: *const V = value;
let data: *const u8 = data as *const u8;
unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_V) }
}
}
indexer/src/storage/fjalls/base.rs
+116
View File
@@ -0,0 +1,116 @@
use std::{collections::BTreeMap, error, mem, path::Path};
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
TransactionalPartitionHandle,
};
use storable_vec::Value;
use unsafe_slice_serde::UnsafeSliceSerde;
use crate::structs::{Height, Version};
use super::Meta;
pub struct Partition<Key, Value> {
meta: Meta,
keyspace: TransactionalKeyspace,
part: TransactionalPartitionHandle,
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
}
impl<K, V> Partition<K, V>
where
K: Into<Slice> + Ord,
V: Into<Slice> + TryFrom<Slice>,
<V as TryFrom<Slice>>::Error: error::Error + Send + Sync + 'static,
{
pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
let meta = Meta::checked_open(path, version)?;
let keyspace = if let Ok(keyspace) = Self::open_keyspace(path) {
keyspace
} else {
meta.reset()?;
return Self::import(path, version);
};
let part = if let Ok(part) = Self::open_partition_handle(&keyspace) {
part
} else {
drop(keyspace);
meta.reset()?;
return Self::import(path, version);
};
let rtx = keyspace.read_tx();
Ok(Self {
meta,
keyspace,
part,
rtx,
puts: BTreeMap::new(),
})
}
pub fn get(&self, key: &K) -> color_eyre::Result<Option<Value<V>>> {
if let Some(v) = self.puts.get(key) {
Ok(Some(Value::Ref(v)))
} else if let Some(slice) = self.rtx.get(&self.part, key.unsafe_as_slice())? {
Ok(Some(Value::Owned(V::try_from(slice)?)))
} else {
Ok(None)
}
}
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
self.puts.insert(key, value);
}
}
pub fn commit(&mut self, height: Height) -> Result<()> {
if self.has(height) && self.puts.is_empty() {
return Ok(());
}
self.meta.export(self.len(), height)?;
let mut wtx = self.keyspace.write_tx();
mem::take(&mut self.puts)
.into_iter()
.for_each(|(key, value)| wtx.insert(&self.part, key, value));
wtx.commit()?;
self.keyspace.persist(PersistMode::SyncAll)?;
self.rtx = self.keyspace.read_tx();
Ok(())
}
pub fn height(&self) -> Option<&Height> {
self.meta.height()
}
pub fn len(&self) -> usize {
self.meta.len() + self.puts.len()
}
pub fn has(&self, height: Height) -> bool {
self.meta.has(height)
}
pub fn needs(&self, height: Height) -> bool {
self.meta.needs(height)
}
fn open_keyspace(path: &Path) -> Result<TransactionalKeyspace> {
fjall::Config::new(path.join("fjall")).open_transactional()
}
fn open_partition_handle(keyspace: &TransactionalKeyspace) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(
"partition",
PartitionCreateOptions::default().manual_journal_persist(true),
)
}
}
indexer/src/storage/fjalls/meta.rs
+99
View File
@@ -0,0 +1,99 @@
use std::{
fs, io,
path::{Path, PathBuf},
};
use unsafe_slice_serde::UnsafeSliceSerde;
use super::{Height, Version};
pub struct Meta {
pathbuf: PathBuf,
version: Version,
height: Option<Height>,
len: usize,
}
impl Meta {
pub fn checked_open(path: &Path, version: Version) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let is_same_version =
Version::try_from(Self::path_version_(path).as_path()).is_ok_and(|prev_version| version == prev_version);
if !is_same_version {
Self::reset_(path)?;
}
let this = Self {
pathbuf: path.to_owned(),
version,
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
len: Self::read_length_(path)?,
};
this.version.write(&this.path_version())?;
Ok(this)
}
pub fn len(&self) -> usize {
self.len
}
pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> {
self.len = len;
self.write_length()?;
self.height = Some(height);
height.write(&self.path_height())
}
pub fn reset(&self) -> io::Result<()> {
Self::reset_(self.pathbuf.as_path())
}
fn reset_(path: &Path) -> io::Result<()> {
fs::remove_dir_all(path)?;
fs::create_dir(path)
}
fn path_version(&self) -> PathBuf {
Self::path_version_(&self.pathbuf)
}
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
pub fn height(&self) -> Option<&Height> {
self.height.as_ref()
}
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)
}
pub fn has(&self, height: Height) -> bool {
!self.needs(height)
}
fn path_height(&self) -> PathBuf {
Self::path_height_(&self.pathbuf)
}
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
// fn read_length(&self) -> color_eyre::Result<usize> {
// Self::read_length_(&self.pathbuf)
// }
fn read_length_(path: &Path) -> color_eyre::Result<usize> {
Ok(fs::read(Self::path_length(path))
.map(|v| usize::unsafe_try_from_slice(v.as_slice()).cloned().unwrap_or_default())
.unwrap_or_default())
}
fn write_length(&self) -> io::Result<()> {
Self::write_length_(&self.pathbuf, self.len)
}
fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> {
fs::write(Self::path_length(path), len.to_le_bytes())
}
fn path_length(path: &Path) -> PathBuf {
path.join("length")
}
}
indexer/src/storage/fjalls/mod.rs
+193
View File
@@ -0,0 +1,193 @@
use std::{path::Path, thread};
use crate::{structs::Version, AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
mod base;
mod meta;
use base::*;
use meta::*;
pub struct Fjalls {
pub addresshash_to_addressindex: Partition<AddressHash, Addressindex>,
pub blockhash_prefix_to_height: Partition<BlockHashPrefix, Height>,
pub txid_prefix_to_txindex: Partition<TxidPrefix, Txindex>,
}
impl Fjalls {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
Ok(Self {
addresshash_to_addressindex: Partition::import(
&path.join("addresshash_to_addressindex"),
Version::from(1),
)?,
blockhash_prefix_to_height: Partition::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
txid_prefix_to_txindex: Partition::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
})
}
// pub fn rollback_from(
// &mut self,
// _wtx: &mut WriteTransaction,
// _height: Height,
// _exit: &Exit,
// ) -> color_eyre::Result<()> {
// panic!();
// let mut txindex = None;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
// Ok(())
// })?;
// let txindex = txindex.context("txindex to not be none by now")?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
// Ok(())
// })?;
// let txoutindex = Txoutindex::from(txindex);
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// todo!("clear txindexvout_to_txoutindex")
// Ok(())
// }
pub fn min_height(&self) -> Option<Height> {
[
self.addresshash_to_addressindex.height(),
self.blockhash_prefix_to_height.height(),
self.txid_prefix_to_txindex.height(),
]
.into_iter()
.min()
.flatten()
.cloned()
}
pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
thread::scope(|scope| {
let addresshash_to_addressindex_commit_handle =
scope.spawn(|| self.addresshash_to_addressindex.commit(height));
let blockhash_prefix_to_height_commit_handle =
scope.spawn(|| self.blockhash_prefix_to_height.commit(height));
let txid_prefix_to_txindex_commit_handle = scope.spawn(|| self.txid_prefix_to_txindex.commit(height));
addresshash_to_addressindex_commit_handle.join().unwrap()?;
blockhash_prefix_to_height_commit_handle.join().unwrap()?;
txid_prefix_to_txindex_commit_handle.join().unwrap()?;
Ok(())
})
}
// pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) {
// self.addressbytes_prefix_to_addressindex.update_meta(wtx, height);
// self.blockhash_prefix_to_height.update_meta(wtx, height);
// self.txid_prefix_to_txindex.update_meta(wtx, height);
// }
// pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
// thread::scope(|scope| {
// vec![
// scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
// scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
// scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
// ]
// .into_iter()
// .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
// })
// }
}
indexer/src/storage/mod.rs
+9
View File
@@ -0,0 +1,9 @@
// mod canopy;
mod fjalls;
// mod sanakirja;
mod storable_vecs;
// pub use canopy::*;
pub use fjalls::*;
// pub use sanakirja::*;
pub use storable_vecs::*;
indexer/src/storage/sanakirjas/meta.rs
+81
View File
@@ -0,0 +1,81 @@
use std::{
fs, io,
path::{Path, PathBuf},
};
use snkrj::UnitDatabase;
use super::{Height, Version};
pub struct StoreMeta {
pathbuf: PathBuf,
version: Version,
height: Option<Height>,
pub len: usize,
}
impl StoreMeta {
pub fn checked_open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
fs::create_dir_all(path)?;
let is_same_version =
Version::try_from(Self::path_version_(path).as_path()).is_ok_and(|prev_version| version == prev_version);
if !is_same_version {
fs::remove_dir_all(path)?;
fs::create_dir(path)?;
}
let this = Self {
pathbuf: path.to_owned(),
version,
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
len: UnitDatabase::read_length_(path),
};
this.version.write(&this.path_version())?;
Ok(this)
}
#[allow(unused)]
pub fn len(&self) -> usize {
self.len
}
pub fn export(mut self, height: Height) -> Result<(), io::Error> {
self.height = Some(height);
height.write(&self.path_height())?;
UnitDatabase::write_length_(&self.pathbuf, self.len)
}
pub fn path_parts(&self) -> PathBuf {
Self::path_parts_(&self.pathbuf)
}
fn path_parts_(path: &Path) -> PathBuf {
path.join("parts")
}
fn path_version(&self) -> PathBuf {
Self::path_version_(&self.pathbuf)
}
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
pub fn height(&self) -> Option<&Height> {
self.height.as_ref()
}
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)
}
pub fn has(&self, height: Height) -> bool {
!self.needs(height)
}
fn path_height(&self) -> PathBuf {
Self::path_height_(&self.pathbuf)
}
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
}
indexer/src/storage/sanakirjas/mod.rs
+174
View File
@@ -0,0 +1,174 @@
use std::{path::Path, thread};
use crate::{
structs::Version, AddressbytesPrefix, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex, Txoutindex,
};
mod meta;
mod multi;
mod unique;
use meta::*;
use unique::*;
pub struct Stores {
pub addressbytes_prefix_to_addressindex: StoreUnique<AddressbytesPrefix, Addressindex>,
pub blockhash_prefix_to_height: StoreUnique<BlockHashPrefix, Height>,
pub txid_prefix_to_txindex: StoreUnique<TxidPrefix, Txindex>,
}
impl Stores {
pub fn open(path: &Path) -> color_eyre::Result<Self> {
Ok(Self {
addressbytes_prefix_to_addressindex: StoreUnique::open(
&path.join("addressbytes_prefix_to_addressindex"),
Version::from(1),
)?,
blockhash_prefix_to_height: StoreUnique::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
txid_prefix_to_txindex: StoreUnique::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
})
}
// pub fn rollback_from(
// &mut self,
// _wtx: &mut WriteTransaction,
// _height: Height,
// _exit: &Exit,
// ) -> color_eyre::Result<()> {
// panic!();
// let mut txindex = None;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
// Ok(())
// })?;
// let txindex = txindex.context("txindex to not be none by now")?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
// Ok(())
// })?;
// let txoutindex = Txoutindex::from(txindex);
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// todo!("clear txindexvout_to_txoutindex")
// Ok(())
// }
pub fn min_height(&self) -> Option<Height> {
[
self.addressbytes_prefix_to_addressindex.height(),
self.blockhash_prefix_to_height.height(),
self.txid_prefix_to_txindex.height(),
]
.into_iter()
.min()
.flatten()
.cloned()
}
pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
thread::scope(|scope| {
vec![
scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
]
.into_iter()
.try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
})
}
}
indexer/src/storage/sanakirjas/multi.rs
+103
View File
@@ -0,0 +1,103 @@
use std::{array, path::Path, sync::OnceLock};
use rayon::prelude::*;
use snkrj::{DatabaseKey, DatabaseMulti, DatabaseValue};
use super::{Height, StoreMeta, Version};
pub struct StoreMulti<K, V>
where
K: DatabaseKey,
V: DatabaseValue,
{
meta: StoreMeta,
pub parts: [OnceLock<Box<DatabaseMulti<K, V>>>; 256],
}
impl<K, V> StoreMulti<K, V>
where
K: DatabaseKey,
V: DatabaseValue,
{
pub fn open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
let meta = StoreMeta::checked_open(path, version)?;
Ok(Self {
meta,
parts: array::from_fn(|_| OnceLock::new()),
})
}
// pub fn len(&self) -> usize {
// self.meta.len()
// }
fn get_or_init_store(&self, key: &K) -> &DatabaseMulti<K, V> {
self.get_or_init_store_(key.as_ne_byte() as usize)
}
fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseMulti<K, V> {
self.parts[storeindex]
.get_or_init(|| Box::new(DatabaseMulti::open(self.meta.path_parts().join(storeindex.to_string())).unwrap()))
}
fn get_or_init_mut_store(&mut self, key: &K) -> &mut DatabaseMulti<K, V> {
self.get_or_init_store(key);
self.parts
.get_mut(key.as_ne_byte() as usize)
.unwrap()
.get_mut()
.unwrap()
}
#[allow(unused)]
pub fn open_all(&self) {
(0..=(u8::MAX) as usize).for_each(|storeindex| {
self.get_or_init_store_(storeindex);
});
}
#[allow(unused)]
pub fn get(&self, key: &K) -> Result<Option<&V>, snkrj::Error> {
self.get_or_init_store(key).get(key)
}
pub fn insert(&mut self, key: K, value: V) -> Option<V> {
self.meta.len += 1;
self.get_or_init_mut_store(&key).insert(key, value)
}
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.meta.needs(height) {
self.insert(key, value);
}
}
pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
if self.has(height) {
return Ok(());
}
self.meta.export(height)?;
self.parts.into_par_iter().try_for_each(|s| {
if let Some(db) = s.into_inner() {
db.export()
} else {
Ok(())
}
})
}
pub fn height(&self) -> Option<&Height> {
self.meta.height()
}
#[allow(unused)]
pub fn needs(&self, height: Height) -> bool {
self.meta.needs(height)
}
pub fn has(&self, height: Height) -> bool {
self.meta.has(height)
}
}
indexer/src/storage/sanakirjas/unique.rs
+102
View File
@@ -0,0 +1,102 @@
use std::{array, path::Path, sync::OnceLock};
use rayon::prelude::*;
use snkrj::{DatabaseKey, DatabaseUnique, DatabaseValue};
use super::{Height, StoreMeta, Version};
pub struct StoreUnique<K, V>
where
K: DatabaseKey,
V: DatabaseValue,
{
meta: StoreMeta,
pub parts: [OnceLock<Box<DatabaseUnique<K, V>>>; 256],
}
impl<K, V> StoreUnique<K, V>
where
K: DatabaseKey,
V: DatabaseValue,
{
pub fn open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
let meta = StoreMeta::checked_open(path, version)?;
Ok(Self {
meta,
parts: array::from_fn(|_| OnceLock::new()),
})
}
// pub fn len(&self) -> usize {
// self.meta.len()
// }
fn get_or_init_store(&self, key: &K) -> &DatabaseUnique<K, V> {
self.get_or_init_store_(key.as_ne_byte() as usize)
}
fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseUnique<K, V> {
self.parts[storeindex].get_or_init(|| {
Box::new(DatabaseUnique::open(self.meta.path_parts().join(storeindex.to_string())).unwrap())
})
}
fn get_or_init_mut_store(&mut self, key: &K) -> &mut DatabaseUnique<K, V> {
self.get_or_init_store(key);
self.parts
.get_mut(key.as_ne_byte() as usize)
.unwrap()
.get_mut()
.unwrap()
}
#[allow(unused)]
pub fn open_all(&self) {
(0..=(u8::MAX) as usize).for_each(|storeindex| {
self.get_or_init_store_(storeindex);
});
}
pub fn get(&self, key: &K) -> Result<Option<&V>, snkrj::Error> {
self.get_or_init_store(key).get(key)
}
pub fn insert(&mut self, key: K, value: V) -> Option<V> {
self.meta.len += 1;
self.get_or_init_mut_store(&key).insert(key, value)
}
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.meta.needs(height) {
self.insert(key, value);
}
}
pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
if self.has(height) {
return Ok(());
}
self.meta.export(height)?;
self.parts.into_par_iter().try_for_each(|s| {
if let Some(db) = s.into_inner() {
db.export()
} else {
Ok(())
}
})
}
pub fn height(&self) -> Option<&Height> {
self.meta.height()
}
pub fn needs(&self, height: Height) -> bool {
self.meta.needs(height)
}
pub fn has(&self, height: Height) -> bool {
self.meta.has(height)
}
}
indexer/src/storage/storable_vecs/base.rs
+119
View File
@@ -0,0 +1,119 @@
use std::{
fmt::Debug,
fs, io,
ops::{Deref, DerefMut},
path::{Path, PathBuf},
};
use super::{Height, Version};
pub struct StorableVec<I, T> {
height: Option<Height>,
pathbuf: PathBuf,
version: Version,
vec: storable_vec::StorableVec<I, T>,
}
impl<I, T> StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug + Clone,
{
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
fs::create_dir_all(path)?;
let pathbuf = path.to_owned();
let path_vec = Self::path_vec_(path);
let path_version = Self::path_version_(path);
let is_same_version =
Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version);
if !is_same_version {
let _ = fs::remove_file(&path_vec);
let _ = fs::remove_file(&path_version);
let _ = fs::remove_file(Self::path_height_(path));
}
let this = Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
pathbuf,
version,
vec: storable_vec::StorableVec::import(&path_vec)?,
};
this.version.write(&this.path_version())?;
Ok(this)
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
if self.needs(height) {
height.write(&self.path_height())?;
}
self.vec.flush()
}
// fn path_vec(&self) -> PathBuf {
// Self::_path_vec(&self.path)
// }
fn path_vec_(path: &Path) -> PathBuf {
path.join("vec")
}
fn path_version(&self) -> PathBuf {
Self::path_version_(&self.pathbuf)
}
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
pub fn height(&self) -> color_eyre::Result<Height> {
Height::try_from(self.path_height().as_path())
}
fn path_height(&self) -> PathBuf {
Self::path_height_(&self.pathbuf)
}
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)
}
#[allow(unused)]
pub fn has(&self, height: Height) -> bool {
!self.needs(height)
}
}
impl<I, T> Deref for StorableVec<I, T> {
type Target = storable_vec::StorableVec<I, T>;
fn deref(&self) -> &Self::Target {
&self.vec
}
}
impl<I, T> DerefMut for StorableVec<I, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.vec
}
}
pub trait AnyBindexVec {
fn height(&self) -> color_eyre::Result<Height>;
fn flush(&mut self, height: Height) -> io::Result<()>;
}
impl<I, T> AnyBindexVec for StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug + Clone,
{
fn height(&self) -> color_eyre::Result<Height> {
self.height()
}
fn flush(&mut self, height: Height) -> io::Result<()> {
self.flush(height)
}
}
indexer/src/storage/storable_vecs/mod.rs
+473
View File
@@ -0,0 +1,473 @@
use std::{fs, io, path::Path};
use biter::bitcoin::{self, transaction, BlockHash, Txid, Weight};
use exit::Exit;
use rayon::prelude::*;
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Height, P2PK33AddressBytes, P2PK65AddressBytes,
P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, Timestamp, Txindex,
Txinindex, Txoutindex, Version,
};
mod base;
use base::*;
pub struct StorableVecs {
pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
pub addressindex_to_height: StorableVec<Addressindex, Height>,
pub height_to_blockhash: StorableVec<Height, BlockHash>,
pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
pub height_to_first_emptyindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_multisigindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_opreturnindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_pushonlyindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_txindex: StorableVec<Height, Txindex>,
pub height_to_first_txinindex: StorableVec<Height, Txinindex>,
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pk33index: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pk65index: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2shindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2trindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2wpkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2wshindex: StorableVec<Height, Addresstypeindex>,
pub height_to_size: StorableVec<Height, usize>,
pub height_to_timestamp: StorableVec<Height, Timestamp>,
pub height_to_weight: StorableVec<Height, Weight>,
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes>,
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes>,
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes>,
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes>,
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes>,
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes>,
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes>,
pub txindex_to_first_txinindex: StorableVec<Txindex, Txinindex>,
pub txindex_to_first_txoutindex: StorableVec<Txindex, Txoutindex>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_locktime: StorableVec<Txindex, bitcoin::absolute::LockTime>,
pub txindex_to_txid: StorableVec<Txindex, Txid>,
pub txindex_to_txversion: StorableVec<Txindex, transaction::Version>,
pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex>,
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount>,
// Can be computed later:
// pub height_to_date: StorableVec<Height, Date>,
// pub height_to_totalfees: StorableVec<Height, Amount>,
// pub height_to_inputcount: StorableVec<Txindex, u32>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
// pub height_to_last_txindex: StorableVec<Height, Txindex>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
// pub height_to_outputcount: StorableVec<Txindex, u32>,
// pub height_to_txcount: StorableVec<Txindex, u32>,
// pub height_to_subsidy: StorableVec<Txindex, u32>,
// pub height_to_minfeerate: StorableVec<Txindex, Feerate>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
// pub txindex_to_inputcount: StorableVec<Txindex, u32>,
// pub txindex_to_outputcount: StorableVec<Txindex, u32>,
// pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex>,
}
// const UNSAFE_BLOCKS: usize = 100;
impl StorableVecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
addressindex_to_addresstype: StorableVec::import(
&path.join("addressindex_to_addresstype"),
Version::from(1),
)?,
addressindex_to_addresstypeindex: StorableVec::import(
&path.join("addressindex_to_addresstypeindex"),
Version::from(1),
)?,
addressindex_to_height: StorableVec::import(&path.join("addressindex_to_height"), Version::from(1))?,
height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?,
height_to_first_addressindex: StorableVec::import(
&path.join("height_to_first_addressindex"),
Version::from(1),
)?,
height_to_first_emptyindex: StorableVec::import(
&path.join("height_to_first_emptyindex"),
Version::from(1),
)?,
height_to_first_multisigindex: StorableVec::import(
&path.join("height_to_first_multisigindex"),
Version::from(1),
)?,
height_to_first_opreturnindex: StorableVec::import(
&path.join("height_to_first_opreturnindex"),
Version::from(1),
)?,
height_to_first_pushonlyindex: StorableVec::import(
&path.join("height_to_first_pushonlyindex"),
Version::from(1),
)?,
height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"), Version::from(1))?,
height_to_first_txinindex: StorableVec::import(&path.join("height_to_first_txinindex"), Version::from(1))?,
height_to_first_txoutindex: StorableVec::import(
&path.join("height_to_first_txoutindex"),
Version::from(1),
)?,
height_to_first_unknownindex: StorableVec::import(
&path.join("height_to_first_unkownindex"),
Version::from(1),
)?,
height_to_first_p2pk33index: StorableVec::import(
&path.join("height_to_first_p2pk33index"),
Version::from(1),
)?,
height_to_first_p2pk65index: StorableVec::import(
&path.join("height_to_first_p2pk65index"),
Version::from(1),
)?,
height_to_first_p2pkhindex: StorableVec::import(
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
)?,
height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?,
height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?,
height_to_first_p2wpkhindex: StorableVec::import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
)?,
height_to_first_p2wshindex: StorableVec::import(
&path.join("height_to_first_p2wshindex"),
Version::from(1),
)?,
height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?,
height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?,
p2pk33index_to_p2pk33addressbytes: StorableVec::import(
&path.join("p2pk33index_to_p2pk33addressbytes"),
Version::from(1),
)?,
p2pk65index_to_p2pk65addressbytes: StorableVec::import(
&path.join("p2pk65index_to_p2pk65addressbytes"),
Version::from(1),
)?,
p2pkhindex_to_p2pkhaddressbytes: StorableVec::import(
&path.join("p2pkhindex_to_p2pkhaddressbytes"),
Version::from(1),
)?,
p2shindex_to_p2shaddressbytes: StorableVec::import(
&path.join("p2shindex_to_p2shaddressbytes"),
Version::from(1),
)?,
p2trindex_to_p2traddressbytes: StorableVec::import(
&path.join("p2trindex_to_p2traddressbytes"),
Version::from(1),
)?,
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import(
&path.join("p2wpkhindex_to_p2wpkhaddressbytes"),
Version::from(1),
)?,
p2wshindex_to_p2wshaddressbytes: StorableVec::import(
&path.join("p2wshindex_to_p2wshaddressbytes"),
Version::from(1),
)?,
txindex_to_first_txinindex: StorableVec::import(
&path.join("txindex_to_first_txinindex"),
Version::from(1),
)?,
txindex_to_first_txoutindex: StorableVec::import(
&path.join("txindex_to_first_txoutindex"),
Version::from(1),
)?,
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_locktime: StorableVec::import(&path.join("txindex_to_locktime"), Version::from(1))?,
txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?,
txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?,
txinindex_to_txoutindex: StorableVec::import(&path.join("txinindex_to_txoutindex"), Version::from(1))?,
txoutindex_to_addressindex: StorableVec::import(
&path.join("txoutindex_to_addressindex"),
Version::from(1),
)?,
txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"), Version::from(1))?,
})
}
pub fn push_addressbytes_if_needed(
&mut self,
index: Addresstypeindex,
addressbytes: Addressbytes,
) -> storable_vec::Result<()> {
match addressbytes {
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
}
}
pub fn get_addressbytes(
&self,
addresstype: Addresstype,
addresstypeindex: Addresstypeindex,
) -> storable_vec::Result<Option<Addressbytes>> {
Ok(match addresstype {
Addresstype::P2PK65 => self
.p2pk65index_to_p2pk65addressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PK33 => self
.p2pk33index_to_p2pk33addressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PKH => self
.p2pkhindex_to_p2pkhaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2SH => self
.p2shindex_to_p2shaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WPKH => self
.p2wpkhindex_to_p2wpkhaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WSH => self
.p2wshindex_to_p2wshaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2TR => self
.p2trindex_to_p2traddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
_ => unreachable!(),
})
}
#[allow(unused)]
pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> {
panic!();
// let mut txindex = None;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
// Ok(())
// })?;
// let txindex = txindex.context("txindex to not be none by now")?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
// Ok(())
// })?;
// let txoutindex = Txoutindex::from(txindex);
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// Ok(())
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
self.as_mut_slice()
.into_par_iter()
.try_for_each(|vec| vec.flush(height))
}
pub fn min_height(&self) -> color_eyre::Result<Option<Height>> {
Ok(self
.as_slice()
.into_iter()
.map(|vec| vec.height().unwrap_or_default())
.min())
}
pub fn as_slice(&self) -> [&dyn AnyBindexVec; 39] {
[
&self.addressindex_to_addresstype as &dyn AnyBindexVec,
&self.addressindex_to_addresstypeindex,
&self.addressindex_to_height,
&self.height_to_blockhash,
&self.height_to_first_addressindex,
&self.height_to_first_emptyindex,
&self.height_to_first_multisigindex,
&self.height_to_first_opreturnindex,
&self.height_to_first_pushonlyindex,
&self.height_to_first_txindex,
&self.height_to_first_txinindex,
&self.height_to_first_txoutindex,
&self.height_to_first_unknownindex,
&self.height_to_first_p2pk33index,
&self.height_to_first_p2pk65index,
&self.height_to_first_p2pkhindex,
&self.height_to_first_p2shindex,
&self.height_to_first_p2trindex,
&self.height_to_first_p2wpkhindex,
&self.height_to_first_p2wshindex,
&self.height_to_size,
&self.height_to_timestamp,
&self.height_to_weight,
&self.p2pk33index_to_p2pk33addressbytes,
&self.p2pk65index_to_p2pk65addressbytes,
&self.p2pkhindex_to_p2pkhaddressbytes,
&self.p2shindex_to_p2shaddressbytes,
&self.p2trindex_to_p2traddressbytes,
&self.p2wpkhindex_to_p2wpkhaddressbytes,
&self.p2wshindex_to_p2wshaddressbytes,
&self.txindex_to_first_txinindex,
&self.txindex_to_first_txoutindex,
&self.txindex_to_height,
&self.txindex_to_locktime,
&self.txindex_to_txid,
&self.txindex_to_txversion,
&self.txinindex_to_txoutindex,
&self.txoutindex_to_addressindex,
&self.txoutindex_to_amount,
]
}
pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyBindexVec + Send + Sync); 39] {
[
&mut self.addressindex_to_addresstype as &mut (dyn AnyBindexVec + Send + Sync),
&mut self.addressindex_to_addresstypeindex,
&mut self.addressindex_to_height,
&mut self.height_to_blockhash,
&mut self.height_to_first_addressindex,
&mut self.height_to_first_emptyindex,
&mut self.height_to_first_multisigindex,
&mut self.height_to_first_opreturnindex,
&mut self.height_to_first_pushonlyindex,
&mut self.height_to_first_txindex,
&mut self.height_to_first_txinindex,
&mut self.height_to_first_txoutindex,
&mut self.height_to_first_unknownindex,
&mut self.height_to_first_p2pk33index,
&mut self.height_to_first_p2pk65index,
&mut self.height_to_first_p2pkhindex,
&mut self.height_to_first_p2shindex,
&mut self.height_to_first_p2trindex,
&mut self.height_to_first_p2wpkhindex,
&mut self.height_to_first_p2wshindex,
&mut self.height_to_size,
&mut self.height_to_timestamp,
&mut self.height_to_weight,
&mut self.p2pk33index_to_p2pk33addressbytes,
&mut self.p2pk65index_to_p2pk65addressbytes,
&mut self.p2pkhindex_to_p2pkhaddressbytes,
&mut self.p2shindex_to_p2shaddressbytes,
&mut self.p2trindex_to_p2traddressbytes,
&mut self.p2wpkhindex_to_p2wpkhaddressbytes,
&mut self.p2wshindex_to_p2wshaddressbytes,
&mut self.txindex_to_first_txinindex,
&mut self.txindex_to_first_txoutindex,
&mut self.txindex_to_height,
&mut self.txindex_to_locktime,
&mut self.txindex_to_txid,
&mut self.txindex_to_txversion,
&mut self.txinindex_to_txoutindex,
&mut self.txoutindex_to_addressindex,
&mut self.txoutindex_to_amount,
]
}
}
indexer/src/structs/addressbytes.rs
+194
View File
@@ -0,0 +1,194 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use super::Addresstype;
#[derive(Debug, PartialEq, Eq)]
pub enum Addressbytes {
P2PK65(P2PK65AddressBytes),
P2PK33(P2PK33AddressBytes),
P2PKH(P2PKHAddressBytes),
P2SH(P2SHAddressBytes),
P2WPKH(P2WPKHAddressBytes),
P2WSH(P2WSHAddressBytes),
P2TR(P2TRAddressBytes),
}
impl Addressbytes {
pub fn as_slice(&self) -> &[u8] {
match self {
Addressbytes::P2PK65(bytes) => &bytes[..],
Addressbytes::P2PK33(bytes) => &bytes[..],
Addressbytes::P2PKH(bytes) => &bytes[..],
Addressbytes::P2SH(bytes) => &bytes[..],
Addressbytes::P2WPKH(bytes) => &bytes[..],
Addressbytes::P2WSH(bytes) => &bytes[..],
Addressbytes::P2TR(bytes) => &bytes[..],
}
}
}
impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes {
type Error = color_eyre::Report;
fn try_from(tuple: (&ScriptBuf, Addresstype)) -> Result<Self, Self::Error> {
let (script, addresstype) = tuple;
match addresstype {
Addresstype::P2PK65 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
67 => &bytes[1..66],
_ => {
dbg!(bytes);
return Err(eyre!("Wrong len"));
}
};
Ok(Self::P2PK65(P2PK65AddressBytes(U8x65::from(bytes))))
}
Addresstype::P2PK33 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
35 => &bytes[1..34],
_ => {
dbg!(bytes);
return Err(eyre!("Wrong len"));
}
};
Ok(Self::P2PK33(P2PK33AddressBytes(U8x33::from(bytes))))
}
Addresstype::P2PKH => {
let bytes = &script.as_bytes()[3..23];
Ok(Self::P2PKH(P2PKHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2SH => {
let bytes = &script.as_bytes()[2..22];
Ok(Self::P2SH(P2SHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2WPKH => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2WPKH(P2WPKHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2WSH => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2WSH(P2WSHAddressBytes(U8x32::from(bytes))))
}
Addresstype::P2TR => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2TR(P2TRAddressBytes(U8x32::from(bytes))))
}
Addresstype::Multisig => Err(eyre!("multisig address type")),
Addresstype::PushOnly => Err(eyre!("push_only address type")),
Addresstype::Unknown => Err(eyre!("unknown address type")),
Addresstype::Empty => Err(eyre!("empty address type")),
Addresstype::OpReturn => Err(eyre!("op_return address type")),
}
}
}
impl From<P2PK65AddressBytes> for Addressbytes {
fn from(value: P2PK65AddressBytes) -> Self {
Self::P2PK65(value)
}
}
impl From<P2PK33AddressBytes> for Addressbytes {
fn from(value: P2PK33AddressBytes) -> Self {
Self::P2PK33(value)
}
}
impl From<P2PKHAddressBytes> for Addressbytes {
fn from(value: P2PKHAddressBytes) -> Self {
Self::P2PKH(value)
}
}
impl From<P2SHAddressBytes> for Addressbytes {
fn from(value: P2SHAddressBytes) -> Self {
Self::P2SH(value)
}
}
impl From<P2WPKHAddressBytes> for Addressbytes {
fn from(value: P2WPKHAddressBytes) -> Self {
Self::P2WPKH(value)
}
}
impl From<P2WSHAddressBytes> for Addressbytes {
fn from(value: P2WSHAddressBytes) -> Self {
Self::P2WSH(value)
}
}
impl From<P2TRAddressBytes> for Addressbytes {
fn from(value: P2TRAddressBytes) -> Self {
Self::P2TR(value)
}
}
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2PK65AddressBytes(U8x65);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2PK33AddressBytes(U8x33);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2PKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2SHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2WPKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2WSHAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2TRAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x20([u8; 20]);
impl From<&[u8]> for U8x20 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 20];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x32([u8; 32]);
impl From<&[u8]> for U8x32 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 32];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x33([u8; 33]);
impl From<&[u8]> for U8x33 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 33];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x64([u8; 64]);
impl From<&[u8]> for U8x64 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 64];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x65([u8; 65]);
impl From<&[u8]> for U8x65 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 65];
arr.copy_from_slice(slice);
Self(arr)
}
}
indexer/src/structs/addressindex.rs
+63
View File
@@ -0,0 +1,63 @@
use derive_deref::{Deref, DerefMut};
// use snkrj::{direct_repr, Storable, UnsizedStorable};
use unsafe_slice_serde::UnsafeSliceSerde;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Addressindex(u32);
// direct_repr!(Addressindex);
impl Addressindex {
pub const BYTES: usize = size_of::<Self>();
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
}
}
impl From<u32> for Addressindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Addressindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Addressindex> for u64 {
fn from(value: Addressindex) -> Self {
value.0 as u64
}
}
impl From<usize> for Addressindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Addressindex> for usize {
fn from(value: Addressindex) -> Self {
value.0 as usize
}
}
impl TryFrom<fjall::Slice> for Addressindex {
type Error = unsafe_slice_serde::Error;
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<Addressindex> for fjall::Slice {
fn from(value: Addressindex) -> Self {
Self::new(value.unsafe_as_slice())
}
}
indexer/src/structs/addresstype.rs
+54
View File
@@ -0,0 +1,54 @@
use biter::bitcoin::ScriptBuf;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Addresstype {
P2PK65,
P2PK33,
P2PKH,
P2SH,
P2WPKH,
P2WSH,
P2TR,
Multisig = 251,
PushOnly = 252,
OpReturn = 253,
Empty = 254,
Unknown = 255,
}
impl From<&ScriptBuf> for Addresstype {
fn from(script: &ScriptBuf) -> Self {
if script.is_p2pk() {
let bytes = script.as_bytes();
match bytes.len() {
67 => Self::P2PK65,
35 => Self::P2PK33,
_ => {
dbg!(bytes);
unreachable!()
}
}
} else if script.is_p2pkh() {
Self::P2PKH
} else if script.is_p2sh() {
Self::P2SH
} else if script.is_p2wpkh() {
Self::P2WPKH
} else if script.is_p2wsh() {
Self::P2WSH
} else if script.is_p2tr() {
Self::P2TR
} else if script.is_empty() {
Self::Empty
} else if script.is_op_return() {
Self::OpReturn
} else if script.is_push_only() {
Self::PushOnly
} else if script.is_multisig() {
Self::Multisig
} else {
Self::Unknown
}
}
}
indexer/src/structs/addresstypeindex.rs
+54
View File
@@ -0,0 +1,54 @@
use derive_deref::{Deref, DerefMut};
// use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Addresstypeindex(u32);
// direct_repr!(Addresstypeindex);
impl Addresstypeindex {
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn clone_then_increment(&mut self) -> Self {
let i = *self;
self.increment();
i
}
}
impl From<u32> for Addresstypeindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Addresstypeindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Addresstypeindex> for u64 {
fn from(value: Addresstypeindex) -> Self {
value.0 as u64
}
}
impl From<usize> for Addresstypeindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Addresstypeindex> for usize {
fn from(value: Addresstypeindex) -> Self {
value.0 as usize
}
}
indexer/src/structs/amount.rs
+90
View File
@@ -0,0 +1,90 @@
use std::{
iter::Sum,
ops::{Add, AddAssign, Mul, Sub, SubAssign},
};
use biter::bitcoin;
use derive_deref::{Deref, DerefMut};
// use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::Height;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Amount(bitcoin::Amount);
// direct_repr!(Amount);
impl Amount {
pub const ZERO: Self = Self(bitcoin::Amount::ZERO);
pub const ONE_BTC_F32: f32 = 100_000_000.0;
pub const ONE_BTC_F64: f64 = 100_000_000.0;
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
}
}
impl Add for Amount {
type Output = Amount;
fn add(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() + rhs.to_sat())
}
}
impl AddAssign for Amount {
fn add_assign(&mut self, rhs: Self) {
*self = Amount::from(self.to_sat() + rhs.to_sat());
}
}
impl Sub for Amount {
type Output = Amount;
fn sub(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() - rhs.to_sat())
}
}
impl SubAssign for Amount {
fn sub_assign(&mut self, rhs: Self) {
*self = Amount::from(self.to_sat() - rhs.to_sat());
}
}
impl Mul<Amount> for Amount {
type Output = Amount;
fn mul(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() * rhs.to_sat())
}
}
impl Mul<u64> for Amount {
type Output = Amount;
fn mul(self, rhs: u64) -> Self::Output {
Amount::from(self.to_sat() * rhs)
}
}
impl Mul<Height> for Amount {
type Output = Amount;
fn mul(self, rhs: Height) -> Self::Output {
Amount::from(self.to_sat() * *rhs as u64)
}
}
impl Sum for Amount {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
let sats: u64 = iter.map(|amt| amt.to_sat()).sum();
Amount::from(sats)
}
}
impl From<u64> for Amount {
fn from(value: u64) -> Self {
Self(bitcoin::Amount::from_sat(value))
}
}
impl From<bitcoin::Amount> for Amount {
fn from(value: bitcoin::Amount) -> Self {
Self(value)
}
}
indexer/src/structs/compressed.rs
+94
View File
@@ -0,0 +1,94 @@
use std::hash::Hasher;
use biter::bitcoin::{BlockHash, Txid};
use derive_deref::Deref;
// use snkrj::{direct_repr, Storable, UnsizedStorable};
use unsafe_slice_serde::UnsafeSliceSerde;
use super::{Addressbytes, Addresstype, SliceExtended};
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct AddressHash([u8; 8]);
// direct_repr!(AddressHash);
impl From<(&Addressbytes, Addresstype)> for AddressHash {
fn from((addressbytes, addresstype): (&Addressbytes, Addresstype)) -> Self {
let mut hasher = rapidhash::RapidHasher::default();
hasher.write(addressbytes.as_slice());
let mut slice = hasher.finish().to_le_bytes();
slice[0] = slice[0].wrapping_add(addresstype as u8);
Self(slice)
}
}
impl From<[u8; 8]> for AddressHash {
fn from(value: [u8; 8]) -> Self {
Self(value)
}
}
impl TryFrom<fjall::Slice> for AddressHash {
type Error = color_eyre::Report;
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<&AddressHash> for fjall::Slice {
fn from(value: &AddressHash) -> Self {
Self::new(value.unsafe_as_slice())
}
}
impl From<AddressHash> for fjall::Slice {
fn from(value: AddressHash) -> Self {
Self::from(&value)
}
}
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct BlockHashPrefix([u8; 8]);
// direct_repr!(BlockHashPrefix);
impl TryFrom<&BlockHash> for BlockHashPrefix {
type Error = color_eyre::Report;
fn try_from(value: &BlockHash) -> Result<Self, Self::Error> {
Ok(Self((&value[..]).read_8x_u8()?))
}
}
impl TryFrom<fjall::Slice> for BlockHashPrefix {
type Error = color_eyre::Report;
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<&BlockHashPrefix> for fjall::Slice {
fn from(value: &BlockHashPrefix) -> Self {
Self::new(value.unsafe_as_slice())
}
}
impl From<BlockHashPrefix> for fjall::Slice {
fn from(value: BlockHashPrefix) -> Self {
Self::from(&value)
}
}
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct TxidPrefix([u8; 8]);
// direct_repr!(TxidPrefix);
impl TryFrom<&Txid> for TxidPrefix {
type Error = color_eyre::Report;
fn try_from(value: &Txid) -> Result<Self, Self::Error> {
Ok(Self((&value[..]).read_8x_u8()?))
}
}
impl TryFrom<fjall::Slice> for TxidPrefix {
type Error = color_eyre::Report;
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<&TxidPrefix> for fjall::Slice {
fn from(value: &TxidPrefix) -> Self {
Self::new(value.unsafe_as_slice())
}
}
impl From<TxidPrefix> for fjall::Slice {
fn from(value: TxidPrefix) -> Self {
Self::from(&value)
}
}
indexer/src/structs/height.rs
+133
View File
@@ -0,0 +1,133 @@
use std::{
fmt, fs, io,
ops::{Add, AddAssign, Rem, Sub},
path::Path,
};
use biter::rpc::{self, RpcApi};
use derive_deref::{Deref, DerefMut};
// use snkrj::{direct_repr, Storable, UnsizedStorable};
use unsafe_slice_serde::UnsafeSliceSerde;
#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct Height(u32);
// direct_repr!(Height);
impl Height {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.unsafe_as_slice())
}
}
impl PartialEq<u64> for Height {
fn eq(&self, other: &u64) -> bool {
**self == *other as u32
}
}
impl Add<u32> for Height {
type Output = Height;
fn add(self, rhs: u32) -> Self::Output {
Self::from(*self + rhs)
}
}
impl Add<usize> for Height {
type Output = Height;
fn add(self, rhs: usize) -> Self::Output {
Self::from(*self + rhs as u32)
}
}
impl Sub<Height> for Height {
type Output = Height;
fn sub(self, rhs: Height) -> Self::Output {
Self::from(*self - *rhs)
}
}
impl Sub<i32> for Height {
type Output = Height;
fn sub(self, rhs: i32) -> Self::Output {
Self::from(*self - rhs as u32)
}
}
impl Sub<u32> for Height {
type Output = Height;
fn sub(self, rhs: u32) -> Self::Output {
Self::from(*self - rhs)
}
}
impl Sub<usize> for Height {
type Output = Height;
fn sub(self, rhs: usize) -> Self::Output {
Self::from(*self - rhs as u32)
}
}
impl AddAssign<usize> for Height {
fn add_assign(&mut self, rhs: usize) {
*self = self.add(rhs);
}
}
impl Rem<usize> for Height {
type Output = Height;
fn rem(self, rhs: usize) -> Self::Output {
Self(self.abs_diff(Height::from(rhs).0))
}
}
impl fmt::Display for Height {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", **self)
}
}
impl From<u32> for Height {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<usize> for Height {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Height> for usize {
fn from(value: Height) -> Self {
value.0 as usize
}
}
impl TryFrom<&Path> for Height {
type Error = color_eyre::Report;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
}
}
impl TryFrom<&rpc::Client> for Height {
type Error = rpc::Error;
fn try_from(value: &rpc::Client) -> Result<Self, Self::Error> {
Ok((value.get_blockchain_info()?.blocks as usize - 1).into())
}
}
impl TryFrom<fjall::Slice> for Height {
type Error = unsafe_slice_serde::Error;
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<Height> for fjall::Slice {
fn from(value: Height) -> Self {
Self::new(value.unsafe_as_slice())
}
}
indexer/src/structs/mod.rs
+31
View File
@@ -0,0 +1,31 @@
mod addressbytes;
mod addressindex;
mod addresstype;
mod addresstypeindex;
mod amount;
mod compressed;
mod height;
mod slice;
mod timestamp;
mod txindex;
mod txinindex;
mod txoutindex;
mod version;
mod vin;
mod vout;
pub use addressbytes::*;
pub use addressindex::*;
pub use addresstype::*;
pub use addresstypeindex::*;
pub use amount::*;
pub use compressed::*;
pub use height::*;
pub use slice::*;
pub use timestamp::*;
pub use txindex::*;
pub use txinindex::*;
pub use txoutindex::*;
pub use version::*;
pub use vin::*;
pub use vout::*;
indexer/src/structs/slice.rs
+55
View File
@@ -0,0 +1,55 @@
use color_eyre::eyre::eyre;
#[allow(unused)]
pub trait SliceExtended {
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]>;
fn read_be_u8(&self) -> color_eyre::Result<u8>;
fn read_be_u16(&self) -> color_eyre::Result<u16>;
fn read_be_u32(&self) -> color_eyre::Result<u32>;
fn read_be_u64(&self) -> color_eyre::Result<u64>;
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()>;
}
impl SliceExtended for &[u8] {
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
(&self[..8]).read_exact(&mut buf)?;
Ok(buf)
}
fn read_be_u8(&self) -> color_eyre::Result<u8> {
let mut buf: [u8; 1] = [0; 1];
self.read_exact(&mut buf)?;
Ok(u8::from_be_bytes(buf))
}
fn read_be_u16(&self) -> color_eyre::Result<u16> {
let mut buf: [u8; 2] = [0; 2];
self.read_exact(&mut buf)?;
Ok(u16::from_be_bytes(buf))
}
fn read_be_u32(&self) -> color_eyre::Result<u32> {
let mut buf: [u8; 4] = [0; 4];
self.read_exact(&mut buf)?;
Ok(u32::from_be_bytes(buf))
}
fn read_be_u64(&self) -> color_eyre::Result<u64> {
let mut buf: [u8; 8] = [0; 8];
self.read_exact(&mut buf)?;
Ok(u64::from_be_bytes(buf))
}
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()> {
let buf_len = buf.len();
if self.len() != buf_len {
dbg!(self.len(), buf_len);
return Err(eyre!("Not exact len"));
}
self.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(())
}
}
indexer/src/structs/timestamp.rs
+11
View File
@@ -0,0 +1,11 @@
use derive_deref::Deref;
#[derive(Debug, Deref, Clone)]
pub struct Timestamp(jiff::Timestamp);
impl TryFrom<u32> for Timestamp {
type Error = jiff::Error;
fn try_from(value: u32) -> Result<Self, Self::Error> {
Ok(Self(jiff::Timestamp::from_second(value as i64)?))
}
}
indexer/src/structs/txindex.rs
+72
View File
@@ -0,0 +1,72 @@
use std::ops::{Add, AddAssign};
use derive_deref::{Deref, DerefMut};
// use snkrj::{direct_repr, Storable, UnsizedStorable};
use unsafe_slice_serde::UnsafeSliceSerde;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Txindex(u32);
// direct_repr!(Txindex);
impl Txindex {
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl Add<Txindex> for Txindex {
type Output = Self;
fn add(self, rhs: Txindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl AddAssign<Txindex> for Txindex {
fn add_assign(&mut self, rhs: Txindex) {
self.0 += rhs.0
}
}
impl From<u32> for Txindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Txindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Txindex> for u64 {
fn from(value: Txindex) -> Self {
value.0 as u64
}
}
impl From<usize> for Txindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Txindex> for usize {
fn from(value: Txindex) -> Self {
value.0 as usize
}
}
impl TryFrom<fjall::Slice> for Txindex {
type Error = unsafe_slice_serde::Error;
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<Txindex> for fjall::Slice {
fn from(value: Txindex) -> Self {
Self::new(value.unsafe_as_slice())
}
}
indexer/src/structs/txinindex.rs
+62
View File
@@ -0,0 +1,62 @@
use std::ops::{Add, AddAssign};
use derive_deref::{Deref, DerefMut};
// use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::Vout;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Txinindex(u64);
// direct_repr!(Txinindex);
impl Txinindex {
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl Add<Txinindex> for Txinindex {
type Output = Self;
fn add(self, rhs: Txinindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl Add<Vout> for Txinindex {
type Output = Self;
fn add(self, rhs: Vout) -> Self::Output {
Self(self.0 + u64::from(rhs))
}
}
impl AddAssign<Txinindex> for Txinindex {
fn add_assign(&mut self, rhs: Txinindex) {
self.0 += rhs.0
}
}
impl From<u64> for Txinindex {
fn from(value: u64) -> Self {
Self(value)
}
}
impl From<Txinindex> for u64 {
fn from(value: Txinindex) -> Self {
value.0
}
}
impl From<usize> for Txinindex {
fn from(value: usize) -> Self {
Self(value as u64)
}
}
impl From<Txinindex> for usize {
fn from(value: Txinindex) -> Self {
value.0 as usize
}
}
indexer/src/structs/txoutindex.rs
+68
View File
@@ -0,0 +1,68 @@
use std::ops::{Add, AddAssign};
use derive_deref::{Deref, DerefMut};
// use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::Vout;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Txoutindex(u64);
// direct_repr!(Txoutindex);
impl Txoutindex {
pub const COINBASE: Self = Self(u64::MAX);
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn is_coinbase(self) -> bool {
self == Self::COINBASE
}
}
impl Add<Txoutindex> for Txoutindex {
type Output = Self;
fn add(self, rhs: Txoutindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl Add<Vout> for Txoutindex {
type Output = Self;
fn add(self, rhs: Vout) -> Self::Output {
Self(self.0 + u64::from(rhs))
}
}
impl AddAssign<Txoutindex> for Txoutindex {
fn add_assign(&mut self, rhs: Txoutindex) {
self.0 += rhs.0
}
}
impl From<u64> for Txoutindex {
fn from(value: u64) -> Self {
Self(value)
}
}
impl From<Txoutindex> for u64 {
fn from(value: Txoutindex) -> Self {
value.0
}
}
impl From<usize> for Txoutindex {
fn from(value: usize) -> Self {
Self(value as u64)
}
}
impl From<Txoutindex> for usize {
fn from(value: Txoutindex) -> Self {
value.0 as usize
}
}
indexer/src/structs/version.rs
+37
View File
@@ -0,0 +1,37 @@
use std::{fs, io, path::Path};
use unsafe_slice_serde::UnsafeSliceSerde;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Version(u32);
impl Version {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.unsafe_as_slice())
}
}
impl From<u32> for Version {
fn from(value: u32) -> Self {
Self(value)
}
}
impl TryFrom<&Path> for Version {
type Error = color_eyre::Report;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
}
}
impl TryFrom<fjall::Slice> for Version {
type Error = color_eyre::Report;
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<Version> for fjall::Slice {
fn from(value: Version) -> Self {
Self::new(value.unsafe_as_slice())
}
}
indexer/src/structs/vin.rs
+30
View File
@@ -0,0 +1,30 @@
use derive_deref::Deref;
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Vin(u32);
impl Vin {
const ZERO: Self = Vin(0_u32);
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
}
}
impl From<u32> for Vin {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<usize> for Vin {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Vin> for u64 {
fn from(value: Vin) -> Self {
value.0 as u64
}
}
indexer/src/structs/vout.rs
+30
View File
@@ -0,0 +1,30 @@
use derive_deref::Deref;
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Vout(u32);
impl Vout {
const ZERO: Self = Vout(0_u32);
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
}
}
impl From<u32> for Vout {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<usize> for Vout {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Vout> for u64 {
fn from(value: Vout) -> Self {
value.0 as u64
}
}