Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-20 11:44:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

bitbase: vecdisk

Browse Source
This commit is contained in:
nym21
2025-01-14 12:32:27 +01:00
parent 1212c3627b
commit 82746a0669
22 changed files with 1261 additions and 490 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/crates/bitbase/.gitignore
+1
View File
@@ -1 +1,2 @@
/database
*_bkp
src/crates/bitbase/Cargo.lock
Generated
+10
View File
@@ -76,6 +76,7 @@ dependencies = [
"ctrlc",
"derive_deref",
"fjall",
"memmap2",
"rayon",
]
@@ -587,6 +588,15 @@ version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "memmap2"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
dependencies = [
"libc",
]
[[package]]
name = "min-max-heap"
version = "1.3.0"
src/crates/bitbase/Cargo.toml
+1
View File
@@ -10,4 +10,5 @@ color-eyre = "0.6.3"
ctrlc = "3.4.5"
derive_deref = "1.1.1"
fjall = "2.5.0"
memmap2 = "0.9.5"
rayon = "1.10.0"
src/crates/bitbase/src/main.rs
+221 -215
View File
@@ -1,12 +1,13 @@
use std::{
collections::BTreeMap,
io::{Read, Write},
path::Path,
str::FromStr,
thread::{self},
};
use biter::{
bitcoin::{hashes::Hash, TxIn, TxOut, Txid},
bitcoin::{TxIn, TxOut, Txid},
bitcoincore_rpc::{Auth, Client},
};
@@ -16,18 +17,17 @@ use color_eyre::eyre::{eyre, ContextCompat};
use fjall::{PersistMode, Slice, TransactionalKeyspace, WriteTransaction};
use rayon::prelude::*;
use structs::{
Addressbytes, Addressindex, Addresstxoutindex, Addresstype, Amount, Exit, Height, Partitions, Prefix,
SliceExtended, Txindex, Txoutindex,
Addressbytes, AddressbytesPrefix, Addressindex, Addressindextxoutindex, Addresstype, Addresstypeindex, Amount,
AnyVecdisk, BlockHashPrefix, Exit, Height, Partitions, TxidPrefix, Txindex, Txindexvout, Txoutindex, Vecdisks,
};
// https://github.com/fjall-rs/fjall/discussions/72
// https://github.com/romanz/electrs/blob/master/doc/schema.md
const U16MAX: usize = u16::MAX as usize;
enum TxInOrAddresstxoutindex<'a> {
#[derive(Debug)]
enum TxInOrAddressindextoutindex<'a> {
TxIn(&'a TxIn),
Addresstxoutindex(Addresstxoutindex),
Addressindextoutindex(Addressindextxoutindex),
}
fn main() -> color_eyre::Result<()> {
@@ -41,27 +41,40 @@ fn main() -> color_eyre::Result<()> {
let exit = Exit::new();
let keyspace = fjall::Config::new("./database").open_transactional()?;
let path_database = Path::new("./database");
let mut parts = Partitions::import(&keyspace, &exit)?;
let mut vecdisks = Vecdisks::import(path_database)?;
let keyspace = fjall::Config::new(path_database).open_transactional()?;
let parts = Partitions::import(&keyspace, &exit)?;
let wtx = keyspace.write_tx();
let mut height = parts.start_height();
let mut txindex = wtx
.get(parts.height_to_first_txindex.data(), Slice::from(height))?
.map(Txindex::from)
let mut txindex = vecdisks
.height_to_first_txindex
.get(height)?
.cloned()
.unwrap_or(Txindex::default());
let mut addressindex = wtx
.get(parts.height_to_first_addressindex.data(), Slice::from(height))?
.map(Addressindex::from)
let mut txoutindex = vecdisks
.height_to_first_txoutindex
.get(height)?
.cloned()
.unwrap_or(Txoutindex::default());
let mut addressindex = vecdisks
.height_to_first_addressindex
.get(height)?
.cloned()
.unwrap_or(Addressindex::default());
let export = |keyspace: &TransactionalKeyspace,
mut wtx: WriteTransaction,
parts: &Partitions,
vecdisks: &mut Vecdisks,
height: Height|
-> color_eyre::Result<()> {
parts.udpate_meta(&mut wtx, height);
@@ -69,6 +82,7 @@ fn main() -> color_eyre::Result<()> {
println!("Exporting...");
wtx.commit()?;
keyspace.persist(PersistMode::SyncAll)?;
vecdisks.flush()?;
println!("Export done");
exit.unblock();
Ok(())
@@ -76,7 +90,7 @@ fn main() -> color_eyre::Result<()> {
let mut wtx_opt = Some(wtx);
biter::new(data_dir, Some(height.into()), None, rpc)
biter::new(data_dir, Some(height.into()), Some(400_000), rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
@@ -85,46 +99,52 @@ fn main() -> color_eyre::Result<()> {
let mut wtx = wtx_opt.take().context("option should have wtx")?;
let saved_blockhash_slice_opt = wtx.get(parts.height_to_blockhash.data(), Slice::from(height))?;
if let Some(saved_blockhash_slice) = saved_blockhash_slice_opt {
if blockhash[..] != saved_blockhash_slice[..] {
parts.rollback_from(&mut wtx, height, &exit)?;
} else {
wtx_opt.replace(wtx);
return Ok(());
}
}
// if let Some(saved_blockhash) = vecdisks.height_to_blockhash.get(height)? {
// if &blockhash != saved_blockhash {
// parts.rollback_from(&mut wtx, height, &exit)?;
// } else {
// wtx_opt.replace(wtx);
// return Ok(());
// }
// }
if parts.blockhash_prefix_to_height.needs(height) {
let blockhash_prefix = BlockHashPrefix::from(&blockhash);
if check_collisions {
if let Some(prev_height_slice) =
wtx.get(parts.blockhash_prefix_to_height.data(), blockhash.prefix())?
wtx.get(parts.blockhash_prefix_to_height.data(), blockhash_prefix.clone())?
{
dbg!(blockhash, Height::from(prev_height_slice));
dbg!(blockhash, Height::try_from(prev_height_slice)?);
return Err(eyre!("Collision, expect prefix to need be set yet"));
}
}
wtx.insert(parts.blockhash_prefix_to_height.data(), blockhash.prefix(),Slice::from(height));
wtx.insert(parts.blockhash_prefix_to_height.data(), blockhash_prefix.clone(),Slice::from(height));
}
if parts.height_to_blockhash.needs(height) {
wtx.insert(parts.height_to_blockhash.data(), Slice::from(height), &blockhash[..]);
}
vecdisks.height_to_blockhash.push_if_needed(height, blockhash)?;
vecdisks.height_to_first_txindex.push_if_needed(height, txindex)?;
vecdisks.height_to_first_txoutindex.push_if_needed(height, txoutindex)?;
vecdisks.height_to_first_addressindex.push_if_needed(height, addressindex)?;
if parts.height_to_first_addressindex.needs(height) {
wtx.insert(
parts.height_to_first_addressindex.data(),
Slice::from(height),
Slice::from(addressindex),
);
}
let outputs = block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (Txindex::from(index), vout as u32, txout))
}).collect::<Vec<_>>();
let txlen = block.txdata.len();
let last_txi = txlen - 1;
let tx_len = block.txdata.len();
let outputs_len = outputs.len();
let (txid_prefix_slice_to_txid_and_index_join_handle, txin_or_addresstxoutindex_vec_handle, txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle) = thread::scope(|scope| {
let txid_prefix_slice_to_txid_and_index_handle = scope.spawn(|| -> color_eyre::Result<_> {
let (txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle, txin_or_addressindextxoutindex_vec_handle, txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle) = thread::scope(|scope| {
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle = scope.spawn(|| -> color_eyre::Result<_> {
block
.txdata
.par_iter()
@@ -132,10 +152,14 @@ fn main() -> color_eyre::Result<()> {
.map(|(index, tx)| -> color_eyre::Result<_> {
let txid = tx.compute_txid();
// Could be removed as should only trigger for two txid (duplicates)
let prev_txindex_slice_opt = wtx.get(parts.txid_prefix_to_txindex.data(), txid.prefix())?;
let prev_txindex_slice_opt = if check_collisions {
// Should only find collisions for two txids (duplicates), see below
wtx.get(parts.txid_prefix_to_txindex.data(), TxidPrefix::from(&txid).clone())?.map(Txindex::try_from)
} else {
None
};
Ok((Slice::from(txid.prefix()), (txid, index, prev_txindex_slice_opt)))
Ok((TxidPrefix::from(&txid), (txid, Txindex::from(index), prev_txindex_slice_opt)))
})
.try_fold(
BTreeMap::new,
@@ -155,7 +179,7 @@ fn main() -> color_eyre::Result<()> {
}
})});
let txin_or_addresstxoutindex_vec_handle = scope.spawn(|| -> color_eyre::Result<Vec<TxInOrAddresstxoutindex>> {
let txin_or_addressindextxoutindex_vec_handle = scope.spawn(|| -> color_eyre::Result<Vec<TxInOrAddressindextoutindex>> {
block
.txdata
.par_iter()
@@ -163,45 +187,50 @@ fn main() -> color_eyre::Result<()> {
.flat_map(|tx| &tx.input)
.map(|txin| -> color_eyre::Result<_> {
let outpoint = txin.previous_output;
let txid_prefix = outpoint.txid.prefix();
let vout = outpoint.vout as u16;
let txid = outpoint.txid;
let vout = outpoint.vout;
let txindex = if let Some(txindex) = wtx
.get(parts.txid_prefix_to_txindex.data(), txid_prefix)?
.map(Txindex::from)
let txindex_local = if let Some(txindex_local) = wtx
.get(parts.txid_prefix_to_txindex.data(), TxidPrefix::from(&txid).clone())?
.map(Txindex::try_from)
{
txindex
txindex_local
} else {
return Ok(TxInOrAddresstxoutindex::TxIn(txin));
};
return Ok(TxInOrAddressindextoutindex::TxIn(txin));
}?;
let txoutindex = Txoutindex::from((txindex, vout));
let txindexvout = Txindexvout::from((txindex_local, vout));
let addressindex = Addressindex::from(
wtx.get(parts.txoutindex_to_addressindex.data(), Slice::from(txoutindex))?
.context("Expect addressindex to not be none")
let txoutindex = Txoutindex::try_from(
wtx.get(parts.txindexvout_to_txoutindex.data(), Slice::from(txindexvout))?
.context("Expect txoutindex to not be none")
.inspect_err(|_| {
let height = Height::from(
wtx.get(parts.txindex_to_height.data(), Slice::from(txindex))
.expect("txindex_to_height get not fail")
.expect("Expect height for txindex"),
);
dbg!(outpoint.txid, txindex, vout, txoutindex, height);
// let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail")
// .expect("Expect height for txindex");
dbg!(outpoint.txid, txindex_local, vout, txindexvout);
})?,
);
)?;
Ok(TxInOrAddresstxoutindex::Addresstxoutindex(Addresstxoutindex::from((
let addressindex = *vecdisks.txoutindex_to_addressindex.get(txoutindex)?
.context("Expect addressindex to not be none")
.inspect_err(|_| {
// let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail")
// .expect("Expect height for txindex");
dbg!(outpoint.txid, txindex_local, vout, txindexvout);
})?;
Ok(TxInOrAddressindextoutindex::Addressindextoutindex(Addressindextxoutindex::from((
addressindex,
txoutindex,
))))
})
.try_fold(
Vec::new,
|mut vec, addresstxoutindex| {
|mut vec, addressindextxoutindex| {
// There is no need to check for bad_tx as there are only 2 instances known
// Which you can find below and which are coinbase tx and thus which are already filtered
if parts.addresstxoutindexes_out.needs(height) {
vec.push(addresstxoutindex?);
if parts.addressindextxoutindex_out.needs(height) {
vec.push(addressindextxoutindex?);
}
Ok(vec)
@@ -219,26 +248,13 @@ fn main() -> color_eyre::Result<()> {
});
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| -> color_eyre::Result<BTreeMap<Txoutindex,
(&TxOut, Addresstype, color_eyre::Result<Addressbytes>, Option<Slice>)>> {
block
.txdata
.par_iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.par_iter()
.enumerate()
.map(move |(vout, txout)| (index, vout, txout))
})
(&TxOut, Txindexvout, Addresstype, color_eyre::Result<Addressbytes>, Option<Slice>)>> {
outputs.into_par_iter().enumerate()
.map(
|(index, vout, txout)| {
if vout > U16MAX {
return Err(eyre!("vout bigger than u16::MAX"));
}
let vout = vout as u16;
let txindex = txindex + index;
let txoutindex = Txoutindex::from((txindex, vout));
|(block_txoutindex, (block_txindex, vout, txout))| {
let txindex_local = txindex + block_txindex;
let txindexvout = Txindexvout::from((txindex_local, vout));
let txoutindex_local = txoutindex + Txoutindex::from(block_txoutindex);
let script = &txout.script_pubkey;
@@ -249,9 +265,10 @@ fn main() -> color_eyre::Result<()> {
});
let addressindex_slice_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
let prefix = AddressbytesPrefix::from(addressbytes);
wtx.get(
parts.addressbytes_prefix_to_addressindex.data(),
Slice::from(addressbytes),
prefix.clone(),
)
.ok()
.and_then(|s| s)
@@ -261,9 +278,10 @@ fn main() -> color_eyre::Result<()> {
if check_collisions && is_new_address {
if let Ok(addressbytes) = &addressbytes_res {
let prefix = AddressbytesPrefix::from(addressbytes);
if let Some(prev) = wtx.get(
parts.addressbytes_prefix_to_addressindex.data(),
Slice::from(addressbytes),
prefix.clone(),
)? {
dbg!(prev);
return Err(eyre!("addressbytes_prefix_to_addressindex collision, expect none"));
@@ -272,8 +290,8 @@ fn main() -> color_eyre::Result<()> {
}
Ok((
txoutindex,
(txout, addresstype, addressbytes_res, addressindex_slice_opt),
txoutindex_local,
(txout, txindexvout, addresstype, addressbytes_res, addressindex_slice_opt),
))
},
)
@@ -296,166 +314,147 @@ fn main() -> color_eyre::Result<()> {
})
});
(txid_prefix_slice_to_txid_and_index_handle.join(), txin_or_addresstxoutindex_vec_handle.join(), txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.join())
(txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(), txin_or_addressindextxoutindex_vec_handle.join(), txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.join())
});
let txid_prefix_slice_to_txid_and_index = txid_prefix_slice_to_txid_and_index_join_handle.ok().context("Expect txid_prefix_slice_to_txid_and_index_join_handle to join")??;
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle.ok().context("Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join")??;
let txin_or_addresstxoutindex_vec = txin_or_addresstxoutindex_vec_handle.ok().context("Export txin_or_addresstxoutindex_vec_handle to join")??;
let txin_or_addressindextxoutindex_vec = txin_or_addressindextxoutindex_vec_handle.ok().context("Export txin_or_addressindextxoutindex_vec_handle to join")??;
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt = txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.ok().context("Expect txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle to join")??;
let mut new_txoutindex_to_addressindex: BTreeMap<Txoutindex, Addressindex> = BTreeMap::new();
let mut new_txindexvout_to_addressindextxoutindex: BTreeMap<Txindexvout, Addressindextxoutindex> = BTreeMap::new();
txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt
.into_iter()
.try_for_each(|(txoutindex, (txout, addresstype, addressbytes_res, addressindex_slice_opt))| -> color_eyre::Result<()> {
.try_for_each(|(txoutindex, (txout, txindexvout, addresstype, addressbytes_res, addressindex_slice_opt))| -> color_eyre::Result<()> {
let amount = Amount::from(txout.value);
let mut addressindex_local = addressindex;
if amount.is_zero() {
if parts.zero_txoutindexes.needs(height) {
wtx.insert(
parts.zero_txoutindexes.data(),
Slice::from(txoutindex),
Slice::default(),
);
}
} else if parts.txoutindex_to_amount.needs(height) {
if parts.txindexvout_to_txoutindex.needs(height) {
wtx.insert(
parts.txoutindex_to_amount.data(),
parts.txindexvout_to_txoutindex.data(),
Slice::from(txindexvout),
Slice::from(txoutindex),
Slice::from(amount),
);
}
if let Some(addressindex_slice) = addressindex_slice_opt {
addressindex_local = addressindex_slice.into()
} else {
new_txoutindex_to_addressindex.insert(txoutindex, addressindex_local);
vecdisks.txoutindex_to_amount.push_if_needed(
txoutindex,
amount,
)?;
if parts.addressindex_to_addresstype.needs(height) {
wtx.insert(
parts.addressindex_to_addresstype.data(),
Slice::from(addressindex_local),
Slice::from(addresstype),
);
}
let mut addressindex_local = addressindex;
if let Some(addressindex_slice) = addressindex_slice_opt {
addressindex_local = Addressindex::try_from(addressindex_slice)?;
} else {
vecdisks.addressindex_to_addresstype.push_if_needed(addressindex_local, addresstype)?;
// TODO: Create counter of other addresstypes instead
let addresstypeindex = Addresstypeindex::from(vecdisks.addresstype_to_addressvecdisk(addresstype).map_or(0, |vecdisk| vecdisk.len()));
vecdisks.addressindex_to_addresstypeindex.push_if_needed(addressindex_local, addresstypeindex)?;
if let Ok(addressbytes) = addressbytes_res {
if parts.addressbytes_prefix_to_addressindex.needs(height) {
wtx.insert(
parts.addressbytes_prefix_to_addressindex.data(),
Slice::from(addressbytes.prefix()),
AddressbytesPrefix::from(&addressbytes).clone(),
Slice::from(addressindex_local),
);
}
if parts.addressindex_to_addressbytes.needs(height) {
wtx.insert(
parts.addressindex_to_addressbytes.data(),
Slice::from(addressindex_local),
Slice::from(&addressbytes),
);
}
vecdisks.push_addressbytes_if_needed(addresstypeindex, addressbytes)?;
}
addressindex.increment();
}
if parts.txoutindex_to_addressindex.needs(height) {
wtx.insert(
parts.txoutindex_to_addressindex.data(),
Slice::from(txoutindex),
Slice::from(addressindex_local),
);
}
new_txindexvout_to_addressindextxoutindex.insert(txindexvout, Addressindextxoutindex::from((addressindex_local, txoutindex)));
vecdisks.txoutindex_to_addressindex.push_if_needed(
txoutindex,
addressindex_local,
)?;
if parts.addresstxoutindexes_in.needs(height) {
let addresstxoutindex = Addresstxoutindex::from((addressindex_local, txoutindex));
if parts.addressindextxoutindex_in.needs(height) {
let addressindextxoutindex = Addressindextxoutindex::from((addressindex_local, txoutindex));
wtx.insert(
parts.addresstxoutindexes_in.data(),
Slice::from(addresstxoutindex),
Slice::default(),
parts.addressindextxoutindex_in.data(),
Slice::from(addressindextxoutindex),
Slice::from(&[]),
);
}
Ok(())
})?;
if parts.addresstxoutindexes_out.needs(height) {
txin_or_addresstxoutindex_vec
if parts.addressindextxoutindex_out.needs(height) {
txin_or_addressindextxoutindex_vec
.into_iter()
.map(|txin_or_addresstxoutindex| -> color_eyre::Result<Addresstxoutindex> {
match txin_or_addresstxoutindex {
TxInOrAddresstxoutindex::Addresstxoutindex(addresstxoutindex) => Ok(addresstxoutindex),
TxInOrAddresstxoutindex::TxIn(txin) => {
.map(|txin_or_addressindextxoutindex| -> color_eyre::Result<Addressindextxoutindex> {
match txin_or_addressindextxoutindex {
TxInOrAddressindextoutindex::Addressindextoutindex(addressindextxoutindex) => Ok(addressindextxoutindex),
TxInOrAddressindextoutindex::TxIn(txin) => {
let outpoint = txin.previous_output;
let txid = outpoint.txid;
let vout = outpoint.vout as u16;
let index = txid_prefix_slice_to_txid_and_index
.get(txid.prefix())
let vout = outpoint.vout;
let index = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.get(&TxidPrefix::from(&txid))
.context("txid should be in same block")?.1;
let txindex = txindex + index;
let txoutindex = Txoutindex::from((txindex, vout));
let addressindex = new_txoutindex_to_addressindex
.remove(&txoutindex)
.context("should have found addressindex from same block").inspect_err(|_| {
dbg!(txin, txoutindex);
})?;
let txindex_local = txindex + index;
Ok(Addresstxoutindex::from((addressindex, txoutindex)))
let txindexvout = Txindexvout::from((txindex_local, vout));
new_txindexvout_to_addressindextxoutindex
.remove(&txindexvout)
.context("should have found addressindex from same block").inspect_err(|_| {
dbg!(&new_txindexvout_to_addressindextxoutindex, txin, txindexvout, txid);
})
}
}
})
.try_for_each(|addresstxoutindex| -> color_eyre::Result<()> {
.try_for_each(|addressindextxoutindex| -> color_eyre::Result<()> {
wtx.insert(
parts.addresstxoutindexes_out.data(),
Slice::from(addresstxoutindex?),
Slice::default(),
parts.addressindextxoutindex_out.data(),
Slice::from(addressindextxoutindex?),
Slice::from(&[]),
);
Ok(())
})?;
}
txid_prefix_slice_to_txid_and_index.into_iter().try_for_each(
|(txid_prefix, (txid, index, prev_txindex_slice_opt))| -> color_eyre::Result<()> {
let txindex = txindex + index;
drop(new_txindexvout_to_addressindextxoutindex);
if index == 0 && parts.height_to_first_txindex.needs(height) {
wtx.insert(
parts.height_to_first_txindex.data(),
Slice::from(height),
Slice::from(txindex),
);
}
if index == last_txi && parts.height_to_last_txindex.needs(height) {
wtx.insert(
parts.height_to_last_txindex.data(),
Slice::from(height),
Slice::from(txindex),
);
}
let mut txindex_to_txid: BTreeMap<Txindex, Txid> = BTreeMap::default();
if parts.txindex_to_txid.needs(height) {
wtx.insert(parts.txindex_to_txid.data(), Slice::from(txindex), &txid[..]);
}
txid_prefix_to_txid_and_block_txindex_and_prev_txindex.into_iter().try_for_each(
|(txid_prefix, (txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
let txindex_local = txindex + index;
match prev_txindex_slice_opt {
txindex_to_txid.insert(txindex_local, txid);
match prev_txindex_opt {
None => {
if parts.txid_prefix_to_txindex.needs(height) {
wtx.insert(parts.txid_prefix_to_txindex.data(), txid_prefix, Slice::from(txindex));
wtx.insert(parts.txid_prefix_to_txindex.data(), txid_prefix.clone(), Slice::from(txindex_local));
}
}
Some(prev_txindex_slice) => {
Some(prev_txindex_res) => {
let prev_txindex = prev_txindex_res?;
// In case if we start at an already parsed height
if txindex_local == prev_txindex {
return Ok(())
}
let len = vecdisks.txindex_to_txid.len();
// Ok if `get` is not par as should happen only twice
let prev_txid = Txid::from_slice(
&wtx.get(parts.txindex_to_txid.data(), &prev_txindex_slice)?
.expect("To have txid for txindex"),
)?;
let prev_txid =
vecdisks.txindex_to_txid.get(prev_txindex)?
.context("To have txid for txindex").inspect_err(|_| {
dbg!(txindex_local, txid, len);
})?;
// If another Txid needs to be added to the list
// We need to check that it's also a coinbase tx otherwise par_iter inputs needs to be updated
@@ -464,59 +463,66 @@ fn main() -> color_eyre::Result<()> {
Txid::from_str("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468")?,
];
let is_dup = only_known_dup_txids.contains(&prev_txid);
let is_dup = only_known_dup_txids.contains(prev_txid);
if !is_dup {
let prev_height = Height::from(
wtx.get(parts.txindex_to_height.data(), &prev_txindex_slice)?
.expect("To have height"),
);
let prev_txindex = Txindex::from(prev_txindex_slice);
dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex,);
let prev_height = vecdisks.txindex_to_height.get(prev_txindex)?.expect("To have height");
dbg!(height, txid, txindex_local, prev_height, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
}
}
}
if parts.txindex_to_height.needs(height) {
wtx.insert(
parts.txindex_to_height.data(),
Slice::from(txindex),
Slice::from(height),
);
}
Ok(())
},
)?;
if parts.height_to_last_addressindex.needs(height) {
wtx.insert(
parts.height_to_last_addressindex.data(),
Slice::from(height),
Slice::from(addressindex.decremented()),
);
}
txindex_to_txid.into_iter().try_for_each(|(txindex, txid)| -> color_eyre::Result<()> {
vecdisks.txindex_to_txid.push_if_needed(txindex, txid)?;
vecdisks.txindex_to_height.push_if_needed(txindex, height)?;
Ok(())
})?;
vecdisks.height_to_last_txindex.push_if_needed(height, txindex.decremented())?;
vecdisks.height_to_last_txoutindex.push_if_needed(height, txoutindex.decremented())?;
vecdisks.height_to_last_addressindex.push_if_needed(height, addressindex.decremented())?;
let should_snapshot = _height % 100 == 0 && !exit.active();
if should_snapshot {
export(&keyspace, wtx, &parts, height)?;
export(&keyspace, wtx, &parts, &mut vecdisks, height)?;
wtx_opt.replace(keyspace.write_tx());
} else {
wtx_opt.replace(wtx);
}
txindex += txlen;
txindex += Txindex::from(tx_len);
txoutindex += Txoutindex::from(outputs_len);
Ok(())
})?;
dbg!(i.elapsed());
pause();
let wtx = wtx_opt.take().context("option should have wtx")?;
export(&keyspace, wtx, &parts, height)?;
export(&keyspace, wtx, &parts, &mut vecdisks, height)?;
pause();
dbg!(i.elapsed());
Ok(())
}
fn pause() {
let mut stdin = std::io::stdin();
let mut stdout = std::io::stdout();
// We want the cursor to stay at the end of the line, so we print without a newline and flush manually.
write!(stdout, "Press any key to continue...").unwrap();
stdout.flush().unwrap();
// Read a single byte and discard
let _ = stdin.read(&mut [0u8]).unwrap();
}
src/crates/bitbase/src/structs/addressbytes.rs
+93 -16
View File
@@ -1,12 +1,19 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::Addresstype;
#[derive(Debug, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord)]
pub struct Addressbytes(Slice);
#[derive(Debug)]
pub enum Addressbytes {
P2PK65(P2PK65AddressBytes),
P2PK33(P2PK33AddressBytes),
P2PKH(P2PKHAddressBytes),
P2SH(P2SHAddressBytes),
P2WPKH(P2WPKHAddressBytes),
P2WSH(P2WSHAddressBytes),
P2TR(P2TRAddressBytes),
}
impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes {
type Error = color_eyre::Report;
@@ -14,37 +21,47 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes {
let (script, addresstype) = tuple;
match addresstype {
Addresstype::P2PK => {
Addresstype::P2PK65 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
67 => &bytes[1..66],
_ => {
dbg!(bytes);
return Err(eyre!("Wrong len"));
}
};
Ok(Self::P2PK65(P2PK65AddressBytes(U8x65::from(bytes))))
}
Addresstype::P2PK33 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
35 => &bytes[1..34],
_ => {
dbg!(bytes);
return Err(eyre!("Wrong len"));
}
};
Ok(Self(bytes.into()))
Ok(Self::P2PK33(P2PK33AddressBytes(U8x33::from(bytes))))
}
Addresstype::P2PKH => {
let bytes = &script.as_bytes()[3..23];
Ok(Self(bytes.into()))
Ok(Self::P2PKH(P2PKHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2SH => {
let bytes = &script.as_bytes()[2..22];
Ok(Self(bytes.into()))
Ok(Self::P2SH(P2SHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2WPKH => {
let bytes = &script.as_bytes()[2..];
Ok(Self(bytes.into()))
Ok(Self::P2WPKH(P2WPKHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2WSH => {
let bytes = &script.as_bytes()[2..];
Ok(Self(bytes.into()))
Ok(Self::P2WSH(P2WSHAddressBytes(U8x32::from(bytes))))
}
Addresstype::P2TR => {
let bytes = &script.as_bytes()[2..];
Ok(Self(bytes.into()))
Ok(Self::P2TR(P2TRAddressBytes(U8x32::from(bytes))))
}
Addresstype::Multisig => Err(eyre!("multisig address type")),
Addresstype::PushOnly => Err(eyre!("push_only address type")),
@@ -55,13 +72,73 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes {
}
}
impl From<Slice> for Addressbytes {
fn from(value: Slice) -> Self {
Self(value)
#[derive(Debug, Clone, Deref)]
pub struct P2PK65AddressBytes(U8x65);
#[derive(Debug, Clone, Deref)]
pub struct P2PK33AddressBytes(U8x33);
#[derive(Debug, Clone, Deref)]
pub struct P2PKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref)]
pub struct P2SHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref)]
pub struct P2WPKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref)]
pub struct P2WSHAddressBytes(U8x32);
#[derive(Debug, Clone, Deref)]
pub struct P2TRAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct U8x20([u8; 20]);
impl From<&[u8]> for U8x20 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 20];
arr.copy_from_slice(slice);
Self(arr)
}
}
impl From<&Addressbytes> for Slice {
fn from(value: &Addressbytes) -> Self {
value.0.clone()
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct U8x32([u8; 32]);
impl From<&[u8]> for U8x32 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 32];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct U8x33([u8; 33]);
impl From<&[u8]> for U8x33 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 33];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct U8x64([u8; 64]);
impl From<&[u8]> for U8x64 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 64];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct U8x65([u8; 65]);
impl From<&[u8]> for U8x65 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 65];
arr.copy_from_slice(slice);
Self(arr)
}
}
src/crates/bitbase/src/structs/addressindex.rs
+21 -3
View File
@@ -39,9 +39,27 @@ impl From<Addressindex> for u64 {
}
}
impl From<Slice> for Addressindex {
fn from(slice: Slice) -> Self {
Self(slice.read_u32())
impl From<usize> for Addressindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Addressindex> for usize {
fn from(value: Addressindex) -> Self {
value.0 as usize
}
}
impl TryFrom<Slice> for Addressindex {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Self::try_from(&value[..])
}
}
impl TryFrom<&[u8]> for Addressindex {
type Error = color_eyre::Report;
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
Ok(Self::from(value.read_be_u32()?))
}
}
impl From<Addressindex> for Slice {
src/crates/bitbase/src/structs/addressindextxoutindex.rs
+38
View File
@@ -0,0 +1,38 @@
use fjall::Slice;
use super::{Addressindex, Txoutindex};
#[derive(Debug)]
pub struct Addressindextxoutindex {
addressindex: Addressindex,
txoutindex: Txoutindex,
}
impl From<(Addressindex, Txoutindex)> for Addressindextxoutindex {
fn from(value: (Addressindex, Txoutindex)) -> Self {
Self {
addressindex: value.0,
txoutindex: value.1,
}
}
}
impl From<Addressindextxoutindex> for Slice {
fn from(value: Addressindextxoutindex) -> Self {
let addressindex_slice = Self::from(value.addressindex);
let txindexvout_slice = Self::from(value.txoutindex);
Self::from([addressindex_slice, txindexvout_slice].concat())
}
}
impl TryFrom<Slice> for Addressindextxoutindex {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
let addressindex = Addressindex::try_from(&value[..Addressindex::BYTES])?;
let txindexvout = Txoutindex::try_from(&value[Addressindex::BYTES..])?;
Ok(Self {
addressindex,
txoutindex: txindexvout,
})
}
}
src/crates/bitbase/src/structs/addresstxoutindex.rs
-35
View File
@@ -1,35 +0,0 @@
use fjall::Slice;
use super::{Addressindex, Txoutindex};
pub struct Addresstxoutindex {
addressindex: Addressindex,
txoutindex: Txoutindex,
}
impl From<(Addressindex, Txoutindex)> for Addresstxoutindex {
fn from(value: (Addressindex, Txoutindex)) -> Self {
Self {
addressindex: value.0,
txoutindex: value.1,
}
}
}
impl From<Addresstxoutindex> for Slice {
fn from(value: Addresstxoutindex) -> Self {
let addressindex_slice = Self::from(value.addressindex);
let txoutindex_slice = Self::from(value.txoutindex);
Self::from([addressindex_slice, txoutindex_slice].concat())
}
}
impl From<Slice> for Addresstxoutindex {
fn from(value: Slice) -> Self {
let addressindex = Addressindex::from(Slice::from(&value[..Addressindex::BYTES]));
let txoutindex = Txoutindex::from(Slice::from(&value[Addressindex::BYTES..]));
Self {
addressindex,
txoutindex,
}
}
}
src/crates/bitbase/src/structs/addresstype.rs
+12 -22
View File
@@ -1,12 +1,9 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use fjall::Slice;
use super::SliceExtended;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Addresstype {
P2PK,
P2PK65,
P2PK33,
P2PKH,
P2SH,
P2WPKH,
@@ -22,7 +19,16 @@ pub enum Addresstype {
impl From<&ScriptBuf> for Addresstype {
fn from(script: &ScriptBuf) -> Self {
if script.is_p2pk() {
Self::P2PK
let bytes = script.as_bytes();
match bytes.len() {
67 => Self::P2PK65,
35 => Self::P2PK33,
_ => {
dbg!(bytes);
unreachable!()
}
}
} else if script.is_p2pkh() {
Self::P2PKH
} else if script.is_p2sh() {
@@ -46,19 +52,3 @@ impl From<&ScriptBuf> for Addresstype {
}
}
}
impl TryFrom<Slice> for Addresstype {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
match value.read_u8() {
x if x == Addresstype::P2PK as u8 => Ok(Addresstype::P2PK),
x if x == Addresstype::P2PKH as u8 => Ok(Addresstype::P2PKH),
_ => Err(eyre!("Unknown type")),
}
}
}
impl From<Addresstype> for Slice {
fn from(addresstype: Addresstype) -> Self {
(addresstype as u8).to_be_bytes().into()
}
}
src/crates/bitbase/src/structs/addresstypeindex.rs
+81
View File
@@ -0,0 +1,81 @@
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::SliceExtended;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Addresstypeindex(u32);
impl Addresstypeindex {
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
}
}
impl From<u32> for Addresstypeindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Addresstypeindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Addresstypeindex> for u64 {
fn from(value: Addresstypeindex) -> Self {
value.0 as u64
}
}
impl From<usize> for Addresstypeindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Addresstypeindex> for usize {
fn from(value: Addresstypeindex) -> Self {
value.0 as usize
}
}
impl TryFrom<Slice> for Addresstypeindex {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Self::try_from(&value[..])
}
}
impl TryFrom<&[u8]> for Addresstypeindex {
type Error = color_eyre::Report;
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
Ok(Self::from(value.read_be_u32()?))
}
}
impl From<Addresstypeindex> for Slice {
fn from(value: Addresstypeindex) -> Self {
value.to_be_bytes().into()
}
}
// impl Bytes for Addresstypeindex {
// const SIZE: usize = size_of::<Self>();
// type ByteArray = [u8; Self::SIZE];
// // fn try_from_bytes(bytes: &[u8]) -> color_eyre::Result<Self> {
// // Ok(Self(Self::read_u32(bytes)))
// // }
// fn to_bytes(&self) -> Self::ByteArray {
// self.to_ne_bytes()
// }
// }
src/crates/bitbase/src/structs/height.rs
+50 -35
View File
@@ -12,41 +12,6 @@ use super::SliceExtended;
#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct Height(u32);
impl From<Slice> for Height {
fn from(slice: Slice) -> Self {
Self(slice.read_u32())
}
}
impl From<Height> for Slice {
fn from(value: Height) -> Self {
value.to_be_bytes().into()
}
}
impl From<u32> for Height {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<usize> for Height {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Height> for usize {
fn from(value: Height) -> Self {
value.0 as usize
}
}
impl TryFrom<&bitcoincore_rpc::Client> for Height {
type Error = bitcoincore_rpc::Error;
fn try_from(value: &bitcoincore_rpc::Client) -> Result<Self, Self::Error> {
Ok((value.get_blockchain_info()?.blocks as usize - 1).into())
}
}
impl PartialEq<u64> for Height {
fn eq(&self, other: &u64) -> bool {
**self == *other as u32
@@ -109,3 +74,53 @@ impl fmt::Display for Height {
write!(f, "{}", **self)
}
}
impl TryFrom<Slice> for Height {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(Self::from((&value[..]).read_be_u32()?))
}
}
impl From<Height> for Slice {
fn from(value: Height) -> Self {
value.to_be_bytes().into()
}
}
impl From<u32> for Height {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<usize> for Height {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Height> for usize {
fn from(value: Height) -> Self {
value.0 as usize
}
}
impl TryFrom<&bitcoincore_rpc::Client> for Height {
type Error = bitcoincore_rpc::Error;
fn try_from(value: &bitcoincore_rpc::Client) -> Result<Self, Self::Error> {
Ok((value.get_blockchain_info()?.blocks as usize - 1).into())
}
}
// impl Bytes for Height {
// const SIZE: usize = size_of::<Self>();
// type ByteArray = [u8; Self::SIZE];
// // fn try_from_bytes(bytes: &[u8]) -> color_eyre::Result<Self> {
// // Ok(Self(Self::read_u32(bytes)))
// // }
// fn to_bytes(&self) -> Self::ByteArray {
// self.to_ne_bytes()
// }
// }
src/crates/bitbase/src/structs/mod.rs
+10 -2
View File
@@ -1,7 +1,8 @@
mod addressbytes;
mod addressindex;
mod addresstxoutindex;
mod addressindextxoutindex;
mod addresstype;
mod addresstypeindex;
mod amount;
mod exit;
mod height;
@@ -10,13 +11,17 @@ mod partitions;
mod prefix;
mod slice;
mod txindex;
mod txindexvout;
mod txoutindex;
mod vecdisk;
mod vecdisks;
mod version;
pub use addressbytes::*;
pub use addressindex::*;
pub use addresstxoutindex::*;
pub use addressindextxoutindex::*;
pub use addresstype::*;
pub use addresstypeindex::*;
pub use amount::*;
pub use exit::*;
pub use height::*;
@@ -25,5 +30,8 @@ pub use partitions::*;
pub use prefix::*;
pub use slice::*;
pub use txindex::*;
pub use txindexvout::*;
pub use txoutindex::*;
pub use vecdisk::*;
pub use vecdisks::*;
pub use version::*;
src/crates/bitbase/src/structs/partition.rs
+21 -7
View File
@@ -15,38 +15,52 @@ impl Partition {
pub const VERSION: &str = "version";
pub const HEIGHT: &str = "height";
pub fn import(keyspace: &TransactionalKeyspace, name: &str, version: Version, exit: &Exit) -> Result<Self> {
pub fn import(
keyspace: &TransactionalKeyspace,
name: &str,
version: Version,
exit: &Exit,
) -> color_eyre::Result<Self> {
let data = Self::open_data(keyspace, name)?;
let meta = Self::open_meta(keyspace, name)?;
let mut height = None;
if let Some(height_res) = meta.get(Self::HEIGHT)?.map(Height::try_from) {
height = Some(height_res?);
}
let mut this = Self {
version,
height: meta.get(Self::HEIGHT)?.map(Height::from),
height,
data,
meta,
};
let mut different_version = false;
if let Some(slice) = this.meta.get(Self::VERSION)? {
if version != Version::from(slice) {
this = this.reset(keyspace, name, exit)?;
}
different_version = Version::try_from(slice).map_or(true, |version2| version != version2);
}
if different_version {
this = this.reset(keyspace, name, exit)?;
}
Ok(this)
}
fn open_data(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(&format!("{name}-data"), Self::create_options())
keyspace.open_partition(&format!("{name}_data"), Self::create_options())
}
fn open_meta(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(&format!("{name}-meta"), Self::create_options())
keyspace.open_partition(&format!("{name}_meta"), Self::create_options())
}
fn create_options() -> PartitionCreateOptions {
PartitionCreateOptions::default().manual_journal_persist(true)
}
// TODO: Still needed ?
pub fn is_safe(&self, height: Height) -> bool {
self.height.is_some_and(|self_height| self_height >= height)
}
src/crates/bitbase/src/structs/partitions.rs
+11 -63
View File
@@ -6,22 +6,11 @@ use crate::structs::{Exit, Height, Partition, Version};
pub struct Partitions {
pub addressbytes_prefix_to_addressindex: Partition,
pub addressindex_to_addressbytes: Partition,
pub addressindex_to_addresstype: Partition,
pub addresstxoutindexes_in: Partition,
pub addresstxoutindexes_out: Partition,
pub addressindextxoutindex_in: Partition,
pub addressindextxoutindex_out: Partition,
pub blockhash_prefix_to_height: Partition,
pub height_to_blockhash: Partition,
pub height_to_first_addressindex: Partition,
pub height_to_first_txindex: Partition,
pub height_to_last_addressindex: Partition,
pub height_to_last_txindex: Partition,
pub txid_prefix_to_txindex: Partition,
pub txindex_to_height: Partition,
pub txindex_to_txid: Partition,
pub txoutindex_to_addressindex: Partition,
pub txoutindex_to_amount: Partition,
pub zero_txoutindexes: Partition,
pub txindexvout_to_txoutindex: Partition,
}
const UNSAFE_BLOCKS: usize = 100;
@@ -35,52 +24,21 @@ impl Partitions {
Version::from(1),
exit,
)?,
addressindex_to_addressbytes: Partition::import(
keyspace,
"addressindex_to_addressbytes",
Version::from(1),
exit,
)?,
addressindex_to_addresstype: Partition::import(
keyspace,
"addressindex_to_addresstype",
Version::from(1),
exit,
)?,
addresstxoutindexes_in: Partition::import(keyspace, "addresstxoutindexes_in", Version::from(1), exit)?,
addresstxoutindexes_out: Partition::import(keyspace, "addresstxoutindexes_out", Version::from(1), exit)?,
addressindextxoutindex_in: Partition::import(keyspace, "addresstxoutindexes_in", Version::from(1), exit)?,
addressindextxoutindex_out: Partition::import(keyspace, "addresstxoutindexes_out", Version::from(1), exit)?,
blockhash_prefix_to_height: Partition::import(
keyspace,
"blockhash_prefix_to_height",
Version::from(1),
exit,
)?,
height_to_blockhash: Partition::import(keyspace, "height_to_blockhash", Version::from(1), exit)?,
height_to_first_addressindex: Partition::import(
keyspace,
"height_to_first_addressindex",
Version::from(1),
exit,
)?,
height_to_first_txindex: Partition::import(keyspace, "height_to_first_txindex", Version::from(1), exit)?,
height_to_last_addressindex: Partition::import(
keyspace,
"height_to_last_addressindex",
Version::from(1),
exit,
)?,
height_to_last_txindex: Partition::import(keyspace, "height_to_last_txindex", Version::from(1), exit)?,
txid_prefix_to_txindex: Partition::import(keyspace, "txid_prefix_to_txindex", Version::from(1), exit)?,
txindex_to_height: Partition::import(keyspace, "txindex_to_height", Version::from(1), exit)?,
txindex_to_txid: Partition::import(keyspace, "txindex_to_txid", Version::from(1), exit)?,
txoutindex_to_addressindex: Partition::import(
txindexvout_to_txoutindex: Partition::import(
keyspace,
"txoutindex_to_addressindex",
"txindexvout_to_txoutindex",
Version::from(1),
exit,
)?,
txoutindex_to_amount: Partition::import(keyspace, "txoutindex_to_amount", Version::from(1), exit)?,
zero_txoutindexes: Partition::import(keyspace, "zero_txoutindexes", Version::from(1), exit)?,
})
}
@@ -221,6 +179,7 @@ impl Partitions {
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// todo!("clear txindexvout_to_txoutindex")
// Ok(())
}
@@ -228,22 +187,11 @@ impl Partitions {
fn to_vec(&self) -> Vec<&Partition> {
vec![
&self.addressbytes_prefix_to_addressindex,
&self.addressindex_to_addressbytes,
&self.addressindex_to_addresstype,
&self.addresstxoutindexes_in,
&self.addresstxoutindexes_out,
&self.addressindextxoutindex_in,
&self.addressindextxoutindex_out,
&self.blockhash_prefix_to_height,
&self.height_to_blockhash,
&self.height_to_first_addressindex,
&self.height_to_first_txindex,
&self.height_to_last_addressindex,
&self.height_to_last_txindex,
&self.txid_prefix_to_txindex,
&self.txindex_to_height,
&self.txindex_to_txid,
&self.txoutindex_to_addressindex,
&self.txoutindex_to_amount,
&self.zero_txoutindexes,
&self.txindexvout_to_txoutindex,
]
}
}
src/crates/bitbase/src/structs/prefix.rs
+41 -11
View File
@@ -1,25 +1,55 @@
use biter::bitcoin::{BlockHash, Txid};
use derive_deref::Deref;
use fjall::Slice;
use super::Addressbytes;
pub trait Prefix {
fn prefix(&self) -> &[u8];
#[derive(Debug, Deref, PartialEq, Eq, PartialOrd, Ord)]
pub struct Prefix(Slice);
impl From<&[u8]> for Prefix {
fn from(value: &[u8]) -> Self {
Self(Slice::from(&value[..8]))
}
}
// pub struct Prefix([u8; 8]);
// impl From<&[u8]> for Prefix {
// fn from(value: &[u8]) -> Self {
// let mut buf: [u8; 8] = [0; 8];
// value.iter().take(8).enumerate().for_each(|(i, v)| {
// buf[i] = *v;
// });
// Self(buf)
// }
// }
impl Prefix for Addressbytes {
fn prefix(&self) -> &[u8] {
&self[..8]
#[derive(Debug, Deref)]
pub struct AddressbytesPrefix(Prefix);
impl From<&Addressbytes> for AddressbytesPrefix {
fn from(value: &Addressbytes) -> Self {
Self(Prefix::from(match value {
Addressbytes::P2PK65(bytes) => &bytes[..],
Addressbytes::P2PK33(bytes) => &bytes[..],
Addressbytes::P2PKH(bytes) => &bytes[..],
Addressbytes::P2SH(bytes) => &bytes[..],
Addressbytes::P2WPKH(bytes) => &bytes[..],
Addressbytes::P2WSH(bytes) => &bytes[..],
Addressbytes::P2TR(bytes) => &bytes[..],
}))
}
}
impl Prefix for BlockHash {
fn prefix(&self) -> &[u8] {
&self[..8]
#[derive(Debug, Deref)]
pub struct BlockHashPrefix(Prefix);
impl From<&BlockHash> for BlockHashPrefix {
fn from(value: &BlockHash) -> Self {
Self(Prefix::from(&value[..]))
}
}
impl Prefix for Txid {
fn prefix(&self) -> &[u8] {
&self[..8]
#[derive(Debug, Deref, PartialEq, Eq, PartialOrd, Ord)]
pub struct TxidPrefix(Prefix);
impl From<&Txid> for TxidPrefix {
fn from(value: &Txid) -> Self {
Self(Prefix::from(&value[..]))
}
}
src/crates/bitbase/src/structs/slice.rs
+28 -29
View File
@@ -1,49 +1,48 @@
use std::io::Read;
use fjall::Slice;
use color_eyre::eyre::eyre;
#[allow(unused)]
pub trait SliceExtended {
fn default() -> Self;
fn read_u8(&self) -> u8;
fn read_u16(&self) -> u16;
fn read_u32(&self) -> u32;
fn read_u64(&self) -> u64;
fn read_exact(&self, buf: &mut [u8]);
fn read_be_u8(&self) -> color_eyre::Result<u8>;
fn read_be_u16(&self) -> color_eyre::Result<u16>;
fn read_be_u32(&self) -> color_eyre::Result<u32>;
fn read_be_u64(&self) -> color_eyre::Result<u64>;
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()>;
}
impl SliceExtended for Slice {
fn default() -> Self {
Self::new(&[])
}
fn read_u8(&self) -> u8 {
impl SliceExtended for &[u8] {
fn read_be_u8(&self) -> color_eyre::Result<u8> {
let mut buf: [u8; 1] = [0; 1];
self.read_exact(&mut buf);
u8::from_be_bytes(buf)
self.read_exact(&mut buf)?;
Ok(u8::from_be_bytes(buf))
}
fn read_u16(&self) -> u16 {
fn read_be_u16(&self) -> color_eyre::Result<u16> {
let mut buf: [u8; 2] = [0; 2];
self.read_exact(&mut buf);
u16::from_be_bytes(buf)
self.read_exact(&mut buf)?;
Ok(u16::from_be_bytes(buf))
}
fn read_u32(&self) -> u32 {
fn read_be_u32(&self) -> color_eyre::Result<u32> {
let mut buf: [u8; 4] = [0; 4];
self.read_exact(&mut buf);
u32::from_be_bytes(buf)
self.read_exact(&mut buf)?;
Ok(u32::from_be_bytes(buf))
}
fn read_u64(&self) -> u64 {
fn read_be_u64(&self) -> color_eyre::Result<u64> {
let mut buf: [u8; 8] = [0; 8];
self.read_exact(&mut buf);
u64::from_be_bytes(buf)
self.read_exact(&mut buf)?;
Ok(u64::from_be_bytes(buf))
}
fn read_exact(&self, buf: &mut [u8]) {
self.bytes().take(buf.len()).enumerate().for_each(|(i, r)| {
buf[i] = r.unwrap();
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()> {
let buf_len = buf.len();
if self.len() != buf_len {
dbg!(self.len(), buf_len);
return Err(eyre!("Not exact len"));
}
self.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(())
}
}
src/crates/bitbase/src/structs/txindex.rs
+38 -18
View File
@@ -9,11 +9,26 @@ use super::SliceExtended;
pub struct Txindex(u32);
impl Txindex {
pub const BYTES: usize = size_of::<Self>();
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl Add<Txindex> for Txindex {
type Output = Self;
fn add(self, rhs: Txindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl AddAssign<Txindex> for Txindex {
fn add_assign(&mut self, rhs: Txindex) {
self.0 += rhs.0
}
}
impl From<u32> for Txindex {
@@ -33,9 +48,27 @@ impl From<Txindex> for u64 {
}
}
impl From<Slice> for Txindex {
fn from(slice: Slice) -> Self {
Self(slice.read_u32())
impl From<usize> for Txindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Txindex> for usize {
fn from(value: Txindex) -> Self {
value.0 as usize
}
}
impl TryFrom<Slice> for Txindex {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Self::try_from(&value[..])
}
}
impl TryFrom<&[u8]> for Txindex {
type Error = color_eyre::Report;
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
Ok(Self::from(value.read_be_u32()?))
}
}
impl From<Txindex> for Slice {
@@ -43,16 +76,3 @@ impl From<Txindex> for Slice {
value.to_be_bytes().into()
}
}
impl Add<usize> for Txindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u32)
}
}
impl AddAssign<usize> for Txindex {
fn add_assign(&mut self, rhs: usize) {
self.0 += rhs as u32
}
}
src/crates/bitbase/src/structs/txindexvout.rs
+51
View File
@@ -0,0 +1,51 @@
use fjall::Slice;
use super::{SliceExtended, Txindex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)]
pub struct Txindexvout {
pub txindex: Txindex,
pub vout: u32,
}
const BYTES: usize = size_of::<Txindexvout>();
impl From<Txindex> for Txindexvout {
fn from(value: Txindex) -> Self {
Self {
txindex: value,
vout: 0,
}
}
}
impl From<(Txindex, u32)> for Txindexvout {
fn from(value: (Txindex, u32)) -> Self {
Self {
txindex: value.0,
vout: value.1,
}
}
}
impl From<Txindexvout> for Slice {
fn from(value: Txindexvout) -> Self {
let txindex_slice = Self::from(value.txindex);
let vout_slice = Self::from(value.vout.to_be_bytes());
Self::from([txindex_slice, vout_slice].concat())
}
}
impl TryFrom<Slice> for Txindexvout {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Self::try_from(&value[..])
}
}
impl TryFrom<&[u8]> for Txindexvout {
type Error = color_eyre::Report;
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
let txindex = Txindex::try_from(&value[..BYTES])?;
let vout = (&value[BYTES..]).read_be_u32()?;
Ok(Self { txindex, vout })
}
}
src/crates/bitbase/src/structs/txoutindex.rs
+56 -30
View File
@@ -1,46 +1,72 @@
use std::ops::{Add, AddAssign};
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::{SliceExtended, Txindex};
use super::SliceExtended;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)]
pub struct Txoutindex {
pub txindex: Txindex,
pub vout: u16,
}
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Txoutindex(u64);
impl Txoutindex {
pub const BYTES: usize = size_of::<Self>();
}
pub fn incremented(self) -> Self {
Self(*self + 1)
}
impl From<Txindex> for Txoutindex {
fn from(value: Txindex) -> Self {
Self {
txindex: value,
vout: 0,
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl From<(Txindex, u16)> for Txoutindex {
fn from(value: (Txindex, u16)) -> Self {
Self {
txindex: value.0,
vout: value.1,
}
impl Add<Txoutindex> for Txoutindex {
type Output = Self;
fn add(self, rhs: Txoutindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl AddAssign<Txoutindex> for Txoutindex {
fn add_assign(&mut self, rhs: Txoutindex) {
self.0 += rhs.0
}
}
impl From<u64> for Txoutindex {
fn from(value: u64) -> Self {
Self(value)
}
}
impl From<Txoutindex> for u64 {
fn from(value: Txoutindex) -> Self {
value.0
}
}
impl From<usize> for Txoutindex {
fn from(value: usize) -> Self {
Self(value as u64)
}
}
impl From<Txoutindex> for usize {
fn from(value: Txoutindex) -> Self {
value.0 as usize
}
}
impl TryFrom<Slice> for Txoutindex {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Self::try_from(&value[..])
}
}
impl TryFrom<&[u8]> for Txoutindex {
type Error = color_eyre::Report;
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
Ok(Self::from(value.read_be_u64()?))
}
}
impl From<Txoutindex> for Slice {
fn from(value: Txoutindex) -> Self {
let txindex_slice = Self::from(value.txindex);
let vout_slice = Self::from(value.vout.to_be_bytes());
Self::from([txindex_slice, vout_slice].concat())
}
}
impl From<Slice> for Txoutindex {
fn from(value: Slice) -> Self {
let txindex = Txindex::from(Slice::from(&value[..Txindex::BYTES]));
let vout = Slice::from(&value[Txindex::BYTES..]).read_u16();
Self { txindex, vout }
value.to_be_bytes().into()
}
}
src/crates/bitbase/src/structs/vecdisk.rs
+219
View File
@@ -0,0 +1,219 @@
use std::{
cmp::Ordering,
fmt::Debug,
fs::{File, OpenOptions},
io::{self, Write},
marker::PhantomData,
mem,
ops::Range,
path::Path,
sync::OnceLock,
};
use color_eyre::eyre::{eyre, ContextCompat};
use derive_deref::{Deref, DerefMut};
use memmap2::{Mmap, MmapOptions};
///
/// A Push only vec stored on disk using Mmap
///
/// Reads (imports of Mmap) are lazy
///
/// Stores only raw data without any overhead, and doesn't even have a header (TODO: which it should, at least to Err if wrong endian)
///
/// The file isn't portable for speed reasons (TODO: but could be ?)
///
#[derive(Debug)]
pub struct Vecdisk<I, T> {
file: File,
mmaps: VecLazyMmap,
disk_len: usize,
cache: Vec<T>,
phantom: PhantomData<I>,
}
/// In bytes
const MAX_PAGE_SIZE: usize = 4096;
impl<I, T> Vecdisk<I, T>
where
I: Into<usize>,
T: Sized + Debug,
{
pub const SIZE: usize = size_of::<T>();
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE;
/// In bytes
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE;
pub fn import(path: &Path) -> color_eyre::Result<Self> {
let file = Self::open_file(path)?;
let mut this = Self {
disk_len: Self::byte_index_to_index(file.metadata()?.len() as usize),
file,
mmaps: VecLazyMmap::default(),
cache: vec![],
phantom: PhantomData,
};
this.reset_mmaps();
Ok(this)
}
fn reset_mmaps(&mut self) {
self.mmaps
.reset((self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize);
}
fn open_file(path: &Path) -> Result<File, io::Error> {
OpenOptions::new()
.read(true)
.create(true)
.truncate(false)
.append(true)
.open(path)
}
#[inline]
fn index_to_mmap_index(index: usize) -> usize {
Self::index_to_byte_index(index) / Self::PAGE_SIZE
}
#[inline]
fn index_to_range(index: usize) -> Range<usize> {
let index = Self::index_to_byte_index(index) % Self::PAGE_SIZE;
index..(index + Self::SIZE)
}
#[inline]
fn index_to_byte_index(index: usize) -> usize {
index * Self::SIZE
}
#[inline]
fn byte_index_to_index(byte_index: usize) -> usize {
byte_index / Self::SIZE
}
#[allow(unused)]
#[inline]
pub fn get(&self, index: I) -> color_eyre::Result<Option<&T>> {
self._get(index.into())
}
fn _get(&self, index: usize) -> color_eyre::Result<Option<&T>> {
if self.disk_len == 0 {
Ok(None)
} else if index > self.disk_len - 1 {
Ok(self.cache.get(index - self.disk_len))
} else {
let mmap_index = Self::index_to_mmap_index(index);
let mmap = self
.mmaps
.get(mmap_index)
.context("Expect mmap to be open")?
.get_or_load(MAX_PAGE_SIZE, (mmap_index * Self::PAGE_SIZE) as u64, &self.file);
let range = Self::index_to_range(index);
let src = &mmap[range];
let (prefix, shorts, suffix) = unsafe { src.align_to::<T>() };
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
dbg!(&src, &prefix, &shorts, &suffix);
return Err(eyre!("align_to issue"));
}
Ok(Some(&shorts[0]))
}
}
#[allow(unused)]
pub fn first(&self) -> color_eyre::Result<Option<&T>> {
self._get(0)
}
#[allow(unused)]
pub fn last(&self) -> color_eyre::Result<Option<&T>> {
let len = self.len();
if len == 0 {
return Ok(None);
}
self._get(len - 1)
}
pub fn push(&mut self, value: T) {
self.cache.push(value)
}
pub fn push_if_needed(&mut self, index: I, value: T) -> color_eyre::Result<()> {
let len = self.len();
let index = index.into();
match len.cmp(&index) {
Ordering::Greater => Ok(()),
Ordering::Equal => {
self.push(value);
Ok(())
}
Ordering::Less => {
dbg!(std::any::type_name::<I>(), std::any::type_name::<T>());
dbg!(len, index, value);
Err(eyre!("Index is too high"))
}
}
}
}
pub trait AnyVecdisk {
fn len(&self) -> usize;
fn flush(&mut self) -> color_eyre::Result<()>;
}
impl<I, T> AnyVecdisk for Vecdisk<I, T>
where
I: Into<usize>,
T: Sized + Debug,
{
fn len(&self) -> usize {
self.disk_len + self.cache.len()
}
fn flush(&mut self) -> color_eyre::Result<()> {
self.disk_len += self.cache.len();
self.reset_mmaps();
let mut bytes: Vec<u8> = vec![];
mem::take(&mut self.cache).into_iter().for_each(|v| {
let data: *const T = &v;
let data: *const u8 = data as *const u8;
let slice = unsafe { std::slice::from_raw_parts(data, Self::SIZE) };
bytes.extend_from_slice(slice)
});
self.file.write_all(&bytes)?;
Ok(())
}
}
#[derive(Debug, Default, Deref, DerefMut)]
struct VecLazyMmap(Vec<LazyMmap>);
impl VecLazyMmap {
pub fn reset(&mut self, len: usize) {
self.clear();
self.resize_with(len, Default::default);
}
}
/// Box to reduce the size, would be 24 instead
#[derive(Debug, Default, Deref)]
struct LazyMmap(OnceLock<Box<Mmap>>);
impl LazyMmap {
pub fn get_or_load(&self, len: usize, offset: u64, file: &File) -> &Mmap {
self.get_or_init(|| Box::new(unsafe { MmapOptions::new().len(len).offset(offset).map(file).unwrap() }))
}
}
src/crates/bitbase/src/structs/vecdisks.rs
+248
View File
@@ -0,0 +1,248 @@
use std::{fs, path::Path};
use biter::bitcoin::{BlockHash, Txid};
use color_eyre::eyre::eyre;
use super::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, AnyVecdisk, Exit, Height, P2PK33AddressBytes,
P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes,
Txindex, Txoutindex, Vecdisk,
};
pub struct Vecdisks {
// TODO:
//
// Add
// txindex_to_fees
// height_to_fees
// height_to_utc_date
// height_to_timestamp
//
// NOT the following as because of reorg it's subjective
// date_to_fees
// date_to_first_height
// date_to_last_height
pub addressindex_to_addresstype: Vecdisk<Addressindex, Addresstype>,
pub addressindex_to_addresstypeindex: Vecdisk<Addressindex, Addresstypeindex>,
pub height_to_blockhash: Vecdisk<Height, BlockHash>,
pub height_to_first_addressindex: Vecdisk<Height, Addressindex>,
pub height_to_first_txindex: Vecdisk<Height, Txindex>,
pub height_to_first_txoutindex: Vecdisk<Height, Txoutindex>,
pub height_to_last_addressindex: Vecdisk<Height, Addressindex>,
pub height_to_last_txindex: Vecdisk<Height, Txindex>,
pub height_to_last_txoutindex: Vecdisk<Height, Txoutindex>,
pub p2pk65index_to_p2pk65addressbytes: Vecdisk<Addresstypeindex, P2PK65AddressBytes>,
pub p2pk33index_to_p2pk33addressbytes: Vecdisk<Addresstypeindex, P2PK33AddressBytes>,
pub p2pkhindex_to_p2pkhaddressbytes: Vecdisk<Addresstypeindex, P2PKHAddressBytes>,
pub p2shindex_to_p2shaddressbytes: Vecdisk<Addresstypeindex, P2SHAddressBytes>,
pub p2wpkhindex_to_p2wpkhaddressbytes: Vecdisk<Addresstypeindex, P2WPKHAddressBytes>,
pub p2wshindex_to_p2wshaddressbytes: Vecdisk<Addresstypeindex, P2WSHAddressBytes>,
pub p2trindex_to_p2traddressbytes: Vecdisk<Addresstypeindex, P2TRAddressBytes>,
pub txindex_to_height: Vecdisk<Txindex, Height>,
pub txindex_to_txid: Vecdisk<Txindex, Txid>,
pub txoutindex_to_addressindex: Vecdisk<Txoutindex, Addressindex>,
pub txoutindex_to_amount: Vecdisk<Txoutindex, Amount>,
}
// const UNSAFE_BLOCKS: usize = 100;
impl Vecdisks {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
addressindex_to_addresstype: Vecdisk::import(&path.join("addressindex_to_addresstype"))?,
addressindex_to_addresstypeindex: Vecdisk::import(&path.join("addressindex_to_addresstypeindex"))?,
height_to_blockhash: Vecdisk::import(&path.join("height_to_blockhash"))?,
height_to_first_addressindex: Vecdisk::import(&path.join("height_to_first_addressindex"))?,
height_to_first_txindex: Vecdisk::import(&path.join("height_to_first_txindex"))?,
height_to_first_txoutindex: Vecdisk::import(&path.join("height_to_first_txoutindex"))?,
height_to_last_addressindex: Vecdisk::import(&path.join("height_to_last_addressindex"))?,
height_to_last_txindex: Vecdisk::import(&path.join("height_to_last_txindex"))?,
height_to_last_txoutindex: Vecdisk::import(&path.join("height_to_last_txoutindex"))?,
p2pk65index_to_p2pk65addressbytes: Vecdisk::import(&path.join("p2pk65index_to_p2pk65addressbytes"))?,
p2pk33index_to_p2pk33addressbytes: Vecdisk::import(&path.join("p2pk33index_to_p2pk33addressbytes"))?,
p2pkhindex_to_p2pkhaddressbytes: Vecdisk::import(&path.join("p2pkhindex_to_p2pkhaddressbytes"))?,
p2shindex_to_p2shaddressbytes: Vecdisk::import(&path.join("p2shindex_to_p2shaddressbytes"))?,
p2wpkhindex_to_p2wpkhaddressbytes: Vecdisk::import(&path.join("p2wpkhindex_to_p2wpkhaddressbytes"))?,
p2wshindex_to_p2wshaddressbytes: Vecdisk::import(&path.join("p2wshindex_to_p2wshaddressbytes"))?,
p2trindex_to_p2traddressbytes: Vecdisk::import(&path.join("p2trindex_to_p2traddressbytes"))?,
txindex_to_height: Vecdisk::import(&path.join("txindex_to_height"))?,
txindex_to_txid: Vecdisk::import(&path.join("txindex_to_txid"))?,
txoutindex_to_addressindex: Vecdisk::import(&path.join("txoutindex_to_addressindex"))?,
txoutindex_to_amount: Vecdisk::import(&path.join("txoutindex_to_amount"))?,
})
}
pub fn addresstype_to_addressvecdisk(&self, addresstype: Addresstype) -> color_eyre::Result<&dyn AnyVecdisk> {
match addresstype {
Addresstype::P2PK65 => Ok(&self.p2pk65index_to_p2pk65addressbytes),
Addresstype::P2PK33 => Ok(&self.p2pk33index_to_p2pk33addressbytes),
Addresstype::P2PKH => Ok(&self.p2pkhindex_to_p2pkhaddressbytes),
Addresstype::P2SH => Ok(&self.p2shindex_to_p2shaddressbytes),
Addresstype::P2WPKH => Ok(&self.p2wpkhindex_to_p2wpkhaddressbytes),
Addresstype::P2WSH => Ok(&self.p2wshindex_to_p2wshaddressbytes),
Addresstype::P2TR => Ok(&self.p2trindex_to_p2traddressbytes),
_ => Err(eyre!("wrong address type")),
}
}
pub fn push_addressbytes_if_needed(
&mut self,
index: Addresstypeindex,
addressbytes: Addressbytes,
) -> color_eyre::Result<()> {
match addressbytes {
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
}
}
#[allow(unused)]
pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> {
panic!();
// let mut txindex = None;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
// Ok(())
// })?;
// let txindex = txindex.context("txindex to not be none by now")?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
// Ok(())
// })?;
// let txoutindex = Txoutindex::from(txindex);
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// Ok(())
}
pub fn flush(&mut self) -> color_eyre::Result<()> {
self.as_mut_vec().into_iter().try_for_each(AnyVecdisk::flush)
}
fn as_mut_vec(&mut self) -> Vec<&mut dyn AnyVecdisk> {
vec![
&mut self.addressindex_to_addresstype,
&mut self.addressindex_to_addresstypeindex,
&mut self.height_to_blockhash,
&mut self.height_to_first_addressindex,
&mut self.height_to_first_txindex,
&mut self.height_to_first_txoutindex,
&mut self.height_to_last_addressindex,
&mut self.height_to_last_txindex,
&mut self.height_to_last_txoutindex,
&mut self.p2pk65index_to_p2pk65addressbytes,
&mut self.p2pk33index_to_p2pk33addressbytes,
&mut self.p2pkhindex_to_p2pkhaddressbytes,
&mut self.p2shindex_to_p2shaddressbytes,
&mut self.p2wpkhindex_to_p2wpkhaddressbytes,
&mut self.p2wshindex_to_p2wshaddressbytes,
&mut self.p2trindex_to_p2traddressbytes,
&mut self.txindex_to_height,
&mut self.txindex_to_txid,
&mut self.txoutindex_to_addressindex,
&mut self.txoutindex_to_amount,
]
}
}
src/crates/bitbase/src/structs/version.rs
+10 -4
View File
@@ -12,12 +12,18 @@ impl From<u8> for Version {
}
}
impl From<Slice> for Version {
fn from(slice: Slice) -> Self {
Self(slice.read_u8())
impl TryFrom<Slice> for Version {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Self::try_from(&value[..])
}
}
impl TryFrom<&[u8]> for Version {
type Error = color_eyre::Report;
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
Ok(Self::from(value.read_be_u8()?))
}
}
impl From<Version> for Slice {
fn from(value: Version) -> Self {
value.to_be_bytes().into()