From 82746a066911fa36a74b1b71976de9a3a728199d Mon Sep 17 00:00:00 2001 From: nym21 Date: Tue, 14 Jan 2025 12:32:27 +0100 Subject: [PATCH] bitbase: vecdisk --- src/crates/bitbase/.gitignore | 1 + src/crates/bitbase/Cargo.lock | 10 + src/crates/bitbase/Cargo.toml | 1 + src/crates/bitbase/src/main.rs | 436 +++++++++--------- .../bitbase/src/structs/addressbytes.rs | 109 ++++- .../bitbase/src/structs/addressindex.rs | 24 +- .../src/structs/addressindextxoutindex.rs | 38 ++ .../bitbase/src/structs/addresstxoutindex.rs | 35 -- src/crates/bitbase/src/structs/addresstype.rs | 34 +- .../bitbase/src/structs/addresstypeindex.rs | 81 ++++ src/crates/bitbase/src/structs/height.rs | 85 ++-- src/crates/bitbase/src/structs/mod.rs | 12 +- src/crates/bitbase/src/structs/partition.rs | 28 +- src/crates/bitbase/src/structs/partitions.rs | 74 +-- src/crates/bitbase/src/structs/prefix.rs | 52 ++- src/crates/bitbase/src/structs/slice.rs | 57 ++- src/crates/bitbase/src/structs/txindex.rs | 56 ++- src/crates/bitbase/src/structs/txindexvout.rs | 51 ++ src/crates/bitbase/src/structs/txoutindex.rs | 86 ++-- src/crates/bitbase/src/structs/vecdisk.rs | 219 +++++++++ src/crates/bitbase/src/structs/vecdisks.rs | 248 ++++++++++ src/crates/bitbase/src/structs/version.rs | 14 +- 22 files changed, 1261 insertions(+), 490 deletions(-) create mode 100644 src/crates/bitbase/src/structs/addressindextxoutindex.rs delete mode 100644 src/crates/bitbase/src/structs/addresstxoutindex.rs create mode 100644 src/crates/bitbase/src/structs/addresstypeindex.rs create mode 100644 src/crates/bitbase/src/structs/txindexvout.rs create mode 100644 src/crates/bitbase/src/structs/vecdisk.rs create mode 100644 src/crates/bitbase/src/structs/vecdisks.rs diff --git a/src/crates/bitbase/.gitignore b/src/crates/bitbase/.gitignore index e2647cd44..ef19dbeff 100644 --- a/src/crates/bitbase/.gitignore +++ b/src/crates/bitbase/.gitignore @@ -1 +1,2 @@ /database +*_bkp diff --git a/src/crates/bitbase/Cargo.lock b/src/crates/bitbase/Cargo.lock index 585837238..82340f639 100644 --- a/src/crates/bitbase/Cargo.lock +++ b/src/crates/bitbase/Cargo.lock @@ -76,6 +76,7 @@ dependencies = [ "ctrlc", "derive_deref", "fjall", + "memmap2", "rayon", ] @@ -587,6 +588,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", +] + [[package]] name = "min-max-heap" version = "1.3.0" diff --git a/src/crates/bitbase/Cargo.toml b/src/crates/bitbase/Cargo.toml index a4a033886..659e81017 100644 --- a/src/crates/bitbase/Cargo.toml +++ b/src/crates/bitbase/Cargo.toml @@ -10,4 +10,5 @@ color-eyre = "0.6.3" ctrlc = "3.4.5" derive_deref = "1.1.1" fjall = "2.5.0" +memmap2 = "0.9.5" rayon = "1.10.0" diff --git a/src/crates/bitbase/src/main.rs b/src/crates/bitbase/src/main.rs index ded75007e..b99ecf268 100644 --- a/src/crates/bitbase/src/main.rs +++ b/src/crates/bitbase/src/main.rs @@ -1,12 +1,13 @@ use std::{ collections::BTreeMap, + io::{Read, Write}, path::Path, str::FromStr, thread::{self}, }; use biter::{ - bitcoin::{hashes::Hash, TxIn, TxOut, Txid}, + bitcoin::{TxIn, TxOut, Txid}, bitcoincore_rpc::{Auth, Client}, }; @@ -16,18 +17,17 @@ use color_eyre::eyre::{eyre, ContextCompat}; use fjall::{PersistMode, Slice, TransactionalKeyspace, WriteTransaction}; use rayon::prelude::*; use structs::{ - Addressbytes, Addressindex, Addresstxoutindex, Addresstype, Amount, Exit, Height, Partitions, Prefix, - SliceExtended, Txindex, Txoutindex, + Addressbytes, AddressbytesPrefix, Addressindex, Addressindextxoutindex, Addresstype, Addresstypeindex, Amount, + AnyVecdisk, BlockHashPrefix, Exit, Height, Partitions, TxidPrefix, Txindex, Txindexvout, Txoutindex, Vecdisks, }; // https://github.com/fjall-rs/fjall/discussions/72 // https://github.com/romanz/electrs/blob/master/doc/schema.md -const U16MAX: usize = u16::MAX as usize; - -enum TxInOrAddresstxoutindex<'a> { +#[derive(Debug)] +enum TxInOrAddressindextoutindex<'a> { TxIn(&'a TxIn), - Addresstxoutindex(Addresstxoutindex), + Addressindextoutindex(Addressindextxoutindex), } fn main() -> color_eyre::Result<()> { @@ -41,27 +41,40 @@ fn main() -> color_eyre::Result<()> { let exit = Exit::new(); - let keyspace = fjall::Config::new("./database").open_transactional()?; + let path_database = Path::new("./database"); - let mut parts = Partitions::import(&keyspace, &exit)?; + let mut vecdisks = Vecdisks::import(path_database)?; + + let keyspace = fjall::Config::new(path_database).open_transactional()?; + + let parts = Partitions::import(&keyspace, &exit)?; let wtx = keyspace.write_tx(); let mut height = parts.start_height(); - let mut txindex = wtx - .get(parts.height_to_first_txindex.data(), Slice::from(height))? - .map(Txindex::from) + let mut txindex = vecdisks + .height_to_first_txindex + .get(height)? + .cloned() .unwrap_or(Txindex::default()); - let mut addressindex = wtx - .get(parts.height_to_first_addressindex.data(), Slice::from(height))? - .map(Addressindex::from) + let mut txoutindex = vecdisks + .height_to_first_txoutindex + .get(height)? + .cloned() + .unwrap_or(Txoutindex::default()); + + let mut addressindex = vecdisks + .height_to_first_addressindex + .get(height)? + .cloned() .unwrap_or(Addressindex::default()); let export = |keyspace: &TransactionalKeyspace, mut wtx: WriteTransaction, parts: &Partitions, + vecdisks: &mut Vecdisks, height: Height| -> color_eyre::Result<()> { parts.udpate_meta(&mut wtx, height); @@ -69,6 +82,7 @@ fn main() -> color_eyre::Result<()> { println!("Exporting..."); wtx.commit()?; keyspace.persist(PersistMode::SyncAll)?; + vecdisks.flush()?; println!("Export done"); exit.unblock(); Ok(()) @@ -76,7 +90,7 @@ fn main() -> color_eyre::Result<()> { let mut wtx_opt = Some(wtx); - biter::new(data_dir, Some(height.into()), None, rpc) + biter::new(data_dir, Some(height.into()), Some(400_000), rpc) .iter() .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { println!("Processing block {_height}..."); @@ -85,46 +99,52 @@ fn main() -> color_eyre::Result<()> { let mut wtx = wtx_opt.take().context("option should have wtx")?; - let saved_blockhash_slice_opt = wtx.get(parts.height_to_blockhash.data(), Slice::from(height))?; - if let Some(saved_blockhash_slice) = saved_blockhash_slice_opt { - if blockhash[..] != saved_blockhash_slice[..] { - parts.rollback_from(&mut wtx, height, &exit)?; - } else { - wtx_opt.replace(wtx); - return Ok(()); - } - } + // if let Some(saved_blockhash) = vecdisks.height_to_blockhash.get(height)? { + // if &blockhash != saved_blockhash { + // parts.rollback_from(&mut wtx, height, &exit)?; + // } else { + // wtx_opt.replace(wtx); + // return Ok(()); + // } + // } if parts.blockhash_prefix_to_height.needs(height) { + let blockhash_prefix = BlockHashPrefix::from(&blockhash); + if check_collisions { + if let Some(prev_height_slice) = - wtx.get(parts.blockhash_prefix_to_height.data(), blockhash.prefix())? + wtx.get(parts.blockhash_prefix_to_height.data(), blockhash_prefix.clone())? { - dbg!(blockhash, Height::from(prev_height_slice)); + dbg!(blockhash, Height::try_from(prev_height_slice)?); return Err(eyre!("Collision, expect prefix to need be set yet")); } } - wtx.insert(parts.blockhash_prefix_to_height.data(), blockhash.prefix(),Slice::from(height)); + wtx.insert(parts.blockhash_prefix_to_height.data(), blockhash_prefix.clone(),Slice::from(height)); } - if parts.height_to_blockhash.needs(height) { - wtx.insert(parts.height_to_blockhash.data(), Slice::from(height), &blockhash[..]); - } + vecdisks.height_to_blockhash.push_if_needed(height, blockhash)?; + vecdisks.height_to_first_txindex.push_if_needed(height, txindex)?; + vecdisks.height_to_first_txoutindex.push_if_needed(height, txoutindex)?; + vecdisks.height_to_first_addressindex.push_if_needed(height, addressindex)?; - if parts.height_to_first_addressindex.needs(height) { - wtx.insert( - parts.height_to_first_addressindex.data(), - Slice::from(height), - Slice::from(addressindex), - ); - } + let outputs = block + .txdata + .iter() + .enumerate() + .flat_map(|(index, tx)| { + tx.output + .iter() + .enumerate() + .map(move |(vout, txout)| (Txindex::from(index), vout as u32, txout)) + }).collect::>(); - let txlen = block.txdata.len(); - let last_txi = txlen - 1; + let tx_len = block.txdata.len(); + let outputs_len = outputs.len(); - let (txid_prefix_slice_to_txid_and_index_join_handle, txin_or_addresstxoutindex_vec_handle, txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle) = thread::scope(|scope| { - let txid_prefix_slice_to_txid_and_index_handle = scope.spawn(|| -> color_eyre::Result<_> { + let (txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle, txin_or_addressindextxoutindex_vec_handle, txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle) = thread::scope(|scope| { + let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle = scope.spawn(|| -> color_eyre::Result<_> { block .txdata .par_iter() @@ -132,10 +152,14 @@ fn main() -> color_eyre::Result<()> { .map(|(index, tx)| -> color_eyre::Result<_> { let txid = tx.compute_txid(); - // Could be removed as should only trigger for two txid (duplicates) - let prev_txindex_slice_opt = wtx.get(parts.txid_prefix_to_txindex.data(), txid.prefix())?; + let prev_txindex_slice_opt = if check_collisions { + // Should only find collisions for two txids (duplicates), see below + wtx.get(parts.txid_prefix_to_txindex.data(), TxidPrefix::from(&txid).clone())?.map(Txindex::try_from) + } else { + None + }; - Ok((Slice::from(txid.prefix()), (txid, index, prev_txindex_slice_opt))) + Ok((TxidPrefix::from(&txid), (txid, Txindex::from(index), prev_txindex_slice_opt))) }) .try_fold( BTreeMap::new, @@ -155,7 +179,7 @@ fn main() -> color_eyre::Result<()> { } })}); - let txin_or_addresstxoutindex_vec_handle = scope.spawn(|| -> color_eyre::Result> { + let txin_or_addressindextxoutindex_vec_handle = scope.spawn(|| -> color_eyre::Result> { block .txdata .par_iter() @@ -163,45 +187,50 @@ fn main() -> color_eyre::Result<()> { .flat_map(|tx| &tx.input) .map(|txin| -> color_eyre::Result<_> { let outpoint = txin.previous_output; - let txid_prefix = outpoint.txid.prefix(); - let vout = outpoint.vout as u16; + let txid = outpoint.txid; + let vout = outpoint.vout; - let txindex = if let Some(txindex) = wtx - .get(parts.txid_prefix_to_txindex.data(), txid_prefix)? - .map(Txindex::from) + let txindex_local = if let Some(txindex_local) = wtx + .get(parts.txid_prefix_to_txindex.data(), TxidPrefix::from(&txid).clone())? + .map(Txindex::try_from) { - txindex + txindex_local } else { - return Ok(TxInOrAddresstxoutindex::TxIn(txin)); - }; + return Ok(TxInOrAddressindextoutindex::TxIn(txin)); + }?; - let txoutindex = Txoutindex::from((txindex, vout)); + let txindexvout = Txindexvout::from((txindex_local, vout)); - let addressindex = Addressindex::from( - wtx.get(parts.txoutindex_to_addressindex.data(), Slice::from(txoutindex))? - .context("Expect addressindex to not be none") + let txoutindex = Txoutindex::try_from( + wtx.get(parts.txindexvout_to_txoutindex.data(), Slice::from(txindexvout))? + .context("Expect txoutindex to not be none") .inspect_err(|_| { - let height = Height::from( - wtx.get(parts.txindex_to_height.data(), Slice::from(txindex)) - .expect("txindex_to_height get not fail") - .expect("Expect height for txindex"), - ); - dbg!(outpoint.txid, txindex, vout, txoutindex, height); + // let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail") + // .expect("Expect height for txindex"); + dbg!(outpoint.txid, txindex_local, vout, txindexvout); })?, - ); + )?; - Ok(TxInOrAddresstxoutindex::Addresstxoutindex(Addresstxoutindex::from(( + let addressindex = *vecdisks.txoutindex_to_addressindex.get(txoutindex)? + .context("Expect addressindex to not be none") + .inspect_err(|_| { + // let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail") + // .expect("Expect height for txindex"); + dbg!(outpoint.txid, txindex_local, vout, txindexvout); + })?; + + Ok(TxInOrAddressindextoutindex::Addressindextoutindex(Addressindextxoutindex::from(( addressindex, txoutindex, )))) }) .try_fold( Vec::new, - |mut vec, addresstxoutindex| { + |mut vec, addressindextxoutindex| { // There is no need to check for bad_tx as there are only 2 instances known // Which you can find below and which are coinbase tx and thus which are already filtered - if parts.addresstxoutindexes_out.needs(height) { - vec.push(addresstxoutindex?); + if parts.addressindextxoutindex_out.needs(height) { + vec.push(addressindextxoutindex?); } Ok(vec) @@ -219,26 +248,13 @@ fn main() -> color_eyre::Result<()> { }); let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| -> color_eyre::Result, Option)>> { - block - .txdata - .par_iter() - .enumerate() - .flat_map(|(index, tx)| { - tx.output - .par_iter() - .enumerate() - .map(move |(vout, txout)| (index, vout, txout)) - }) + (&TxOut, Txindexvout, Addresstype, color_eyre::Result, Option)>> { + outputs.into_par_iter().enumerate() .map( - |(index, vout, txout)| { - if vout > U16MAX { - return Err(eyre!("vout bigger than u16::MAX")); - } - - let vout = vout as u16; - let txindex = txindex + index; - let txoutindex = Txoutindex::from((txindex, vout)); + |(block_txoutindex, (block_txindex, vout, txout))| { + let txindex_local = txindex + block_txindex; + let txindexvout = Txindexvout::from((txindex_local, vout)); + let txoutindex_local = txoutindex + Txoutindex::from(block_txoutindex); let script = &txout.script_pubkey; @@ -249,9 +265,10 @@ fn main() -> color_eyre::Result<()> { }); let addressindex_slice_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| { + let prefix = AddressbytesPrefix::from(addressbytes); wtx.get( parts.addressbytes_prefix_to_addressindex.data(), - Slice::from(addressbytes), + prefix.clone(), ) .ok() .and_then(|s| s) @@ -261,9 +278,10 @@ fn main() -> color_eyre::Result<()> { if check_collisions && is_new_address { if let Ok(addressbytes) = &addressbytes_res { + let prefix = AddressbytesPrefix::from(addressbytes); if let Some(prev) = wtx.get( parts.addressbytes_prefix_to_addressindex.data(), - Slice::from(addressbytes), + prefix.clone(), )? { dbg!(prev); return Err(eyre!("addressbytes_prefix_to_addressindex collision, expect none")); @@ -272,8 +290,8 @@ fn main() -> color_eyre::Result<()> { } Ok(( - txoutindex, - (txout, addresstype, addressbytes_res, addressindex_slice_opt), + txoutindex_local, + (txout, txindexvout, addresstype, addressbytes_res, addressindex_slice_opt), )) }, ) @@ -296,166 +314,147 @@ fn main() -> color_eyre::Result<()> { }) }); - (txid_prefix_slice_to_txid_and_index_handle.join(), txin_or_addresstxoutindex_vec_handle.join(), txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.join()) + (txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(), txin_or_addressindextxoutindex_vec_handle.join(), txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.join()) }); - let txid_prefix_slice_to_txid_and_index = txid_prefix_slice_to_txid_and_index_join_handle.ok().context("Expect txid_prefix_slice_to_txid_and_index_join_handle to join")??; + let txid_prefix_to_txid_and_block_txindex_and_prev_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle.ok().context("Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join")??; - let txin_or_addresstxoutindex_vec = txin_or_addresstxoutindex_vec_handle.ok().context("Export txin_or_addresstxoutindex_vec_handle to join")??; + let txin_or_addressindextxoutindex_vec = txin_or_addressindextxoutindex_vec_handle.ok().context("Export txin_or_addressindextxoutindex_vec_handle to join")??; let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt = txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.ok().context("Expect txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle to join")??; - let mut new_txoutindex_to_addressindex: BTreeMap = BTreeMap::new(); + let mut new_txindexvout_to_addressindextxoutindex: BTreeMap = BTreeMap::new(); txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt .into_iter() - .try_for_each(|(txoutindex, (txout, addresstype, addressbytes_res, addressindex_slice_opt))| -> color_eyre::Result<()> { + .try_for_each(|(txoutindex, (txout, txindexvout, addresstype, addressbytes_res, addressindex_slice_opt))| -> color_eyre::Result<()> { let amount = Amount::from(txout.value); - let mut addressindex_local = addressindex; - - if amount.is_zero() { - if parts.zero_txoutindexes.needs(height) { - wtx.insert( - parts.zero_txoutindexes.data(), - Slice::from(txoutindex), - Slice::default(), - ); - } - } else if parts.txoutindex_to_amount.needs(height) { + if parts.txindexvout_to_txoutindex.needs(height) { wtx.insert( - parts.txoutindex_to_amount.data(), + parts.txindexvout_to_txoutindex.data(), + Slice::from(txindexvout), Slice::from(txoutindex), - Slice::from(amount), ); } - if let Some(addressindex_slice) = addressindex_slice_opt { - addressindex_local = addressindex_slice.into() - } else { - new_txoutindex_to_addressindex.insert(txoutindex, addressindex_local); + vecdisks.txoutindex_to_amount.push_if_needed( + txoutindex, + amount, + )?; - if parts.addressindex_to_addresstype.needs(height) { - wtx.insert( - parts.addressindex_to_addresstype.data(), - Slice::from(addressindex_local), - Slice::from(addresstype), - ); - } + let mut addressindex_local = addressindex; + + if let Some(addressindex_slice) = addressindex_slice_opt { + addressindex_local = Addressindex::try_from(addressindex_slice)?; + } else { + vecdisks.addressindex_to_addresstype.push_if_needed(addressindex_local, addresstype)?; + + // TODO: Create counter of other addresstypes instead + let addresstypeindex = Addresstypeindex::from(vecdisks.addresstype_to_addressvecdisk(addresstype).map_or(0, |vecdisk| vecdisk.len())); + + vecdisks.addressindex_to_addresstypeindex.push_if_needed(addressindex_local, addresstypeindex)?; if let Ok(addressbytes) = addressbytes_res { if parts.addressbytes_prefix_to_addressindex.needs(height) { wtx.insert( parts.addressbytes_prefix_to_addressindex.data(), - Slice::from(addressbytes.prefix()), + AddressbytesPrefix::from(&addressbytes).clone(), Slice::from(addressindex_local), ); } - if parts.addressindex_to_addressbytes.needs(height) { - wtx.insert( - parts.addressindex_to_addressbytes.data(), - Slice::from(addressindex_local), - Slice::from(&addressbytes), - ); - } - + vecdisks.push_addressbytes_if_needed(addresstypeindex, addressbytes)?; } addressindex.increment(); } - if parts.txoutindex_to_addressindex.needs(height) { - wtx.insert( - parts.txoutindex_to_addressindex.data(), - Slice::from(txoutindex), - Slice::from(addressindex_local), - ); - } + new_txindexvout_to_addressindextxoutindex.insert(txindexvout, Addressindextxoutindex::from((addressindex_local, txoutindex))); + vecdisks.txoutindex_to_addressindex.push_if_needed( + txoutindex, + addressindex_local, + )?; - if parts.addresstxoutindexes_in.needs(height) { - let addresstxoutindex = Addresstxoutindex::from((addressindex_local, txoutindex)); + if parts.addressindextxoutindex_in.needs(height) { + let addressindextxoutindex = Addressindextxoutindex::from((addressindex_local, txoutindex)); wtx.insert( - parts.addresstxoutindexes_in.data(), - Slice::from(addresstxoutindex), - Slice::default(), + parts.addressindextxoutindex_in.data(), + Slice::from(addressindextxoutindex), + Slice::from(&[]), ); } Ok(()) })?; - if parts.addresstxoutindexes_out.needs(height) { - txin_or_addresstxoutindex_vec + if parts.addressindextxoutindex_out.needs(height) { + txin_or_addressindextxoutindex_vec .into_iter() - .map(|txin_or_addresstxoutindex| -> color_eyre::Result { - match txin_or_addresstxoutindex { - TxInOrAddresstxoutindex::Addresstxoutindex(addresstxoutindex) => Ok(addresstxoutindex), - TxInOrAddresstxoutindex::TxIn(txin) => { + .map(|txin_or_addressindextxoutindex| -> color_eyre::Result { + match txin_or_addressindextxoutindex { + TxInOrAddressindextoutindex::Addressindextoutindex(addressindextxoutindex) => Ok(addressindextxoutindex), + TxInOrAddressindextoutindex::TxIn(txin) => { let outpoint = txin.previous_output; let txid = outpoint.txid; - let vout = outpoint.vout as u16; - let index = txid_prefix_slice_to_txid_and_index - .get(txid.prefix()) + let vout = outpoint.vout; + let index = txid_prefix_to_txid_and_block_txindex_and_prev_txindex + .get(&TxidPrefix::from(&txid)) .context("txid should be in same block")?.1; - let txindex = txindex + index; - let txoutindex = Txoutindex::from((txindex, vout)); - let addressindex = new_txoutindex_to_addressindex - .remove(&txoutindex) - .context("should have found addressindex from same block").inspect_err(|_| { - dbg!(txin, txoutindex); - })?; + let txindex_local = txindex + index; - Ok(Addresstxoutindex::from((addressindex, txoutindex))) + let txindexvout = Txindexvout::from((txindex_local, vout)); + + new_txindexvout_to_addressindextxoutindex + .remove(&txindexvout) + .context("should have found addressindex from same block").inspect_err(|_| { + dbg!(&new_txindexvout_to_addressindextxoutindex, txin, txindexvout, txid); + }) } } }) - .try_for_each(|addresstxoutindex| -> color_eyre::Result<()> { + .try_for_each(|addressindextxoutindex| -> color_eyre::Result<()> { wtx.insert( - parts.addresstxoutindexes_out.data(), - Slice::from(addresstxoutindex?), - Slice::default(), + parts.addressindextxoutindex_out.data(), + Slice::from(addressindextxoutindex?), + Slice::from(&[]), ); Ok(()) })?; } - txid_prefix_slice_to_txid_and_index.into_iter().try_for_each( - |(txid_prefix, (txid, index, prev_txindex_slice_opt))| -> color_eyre::Result<()> { - let txindex = txindex + index; + drop(new_txindexvout_to_addressindextxoutindex); - if index == 0 && parts.height_to_first_txindex.needs(height) { - wtx.insert( - parts.height_to_first_txindex.data(), - Slice::from(height), - Slice::from(txindex), - ); - } - if index == last_txi && parts.height_to_last_txindex.needs(height) { - wtx.insert( - parts.height_to_last_txindex.data(), - Slice::from(height), - Slice::from(txindex), - ); - } + let mut txindex_to_txid: BTreeMap = BTreeMap::default(); - if parts.txindex_to_txid.needs(height) { - wtx.insert(parts.txindex_to_txid.data(), Slice::from(txindex), &txid[..]); - } + txid_prefix_to_txid_and_block_txindex_and_prev_txindex.into_iter().try_for_each( + |(txid_prefix, (txid, index, prev_txindex_opt))| -> color_eyre::Result<()> { + let txindex_local = txindex + index; - match prev_txindex_slice_opt { + txindex_to_txid.insert(txindex_local, txid); + + match prev_txindex_opt { None => { if parts.txid_prefix_to_txindex.needs(height) { - wtx.insert(parts.txid_prefix_to_txindex.data(), txid_prefix, Slice::from(txindex)); + wtx.insert(parts.txid_prefix_to_txindex.data(), txid_prefix.clone(), Slice::from(txindex_local)); } } - Some(prev_txindex_slice) => { + Some(prev_txindex_res) => { + let prev_txindex = prev_txindex_res?; + + // In case if we start at an already parsed height + if txindex_local == prev_txindex { + return Ok(()) + } + + let len = vecdisks.txindex_to_txid.len(); // Ok if `get` is not par as should happen only twice - let prev_txid = Txid::from_slice( - &wtx.get(parts.txindex_to_txid.data(), &prev_txindex_slice)? - .expect("To have txid for txindex"), - )?; + let prev_txid = + vecdisks.txindex_to_txid.get(prev_txindex)? + .context("To have txid for txindex").inspect_err(|_| { + dbg!(txindex_local, txid, len); + })?; // If another Txid needs to be added to the list // We need to check that it's also a coinbase tx otherwise par_iter inputs needs to be updated @@ -464,59 +463,66 @@ fn main() -> color_eyre::Result<()> { Txid::from_str("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468")?, ]; - let is_dup = only_known_dup_txids.contains(&prev_txid); + let is_dup = only_known_dup_txids.contains(prev_txid); if !is_dup { - let prev_height = Height::from( - wtx.get(parts.txindex_to_height.data(), &prev_txindex_slice)? - .expect("To have height"), - ); - let prev_txindex = Txindex::from(prev_txindex_slice); - dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex,); + let prev_height = vecdisks.txindex_to_height.get(prev_txindex)?.expect("To have height"); + dbg!(height, txid, txindex_local, prev_height, prev_txid, prev_txindex); return Err(eyre!("Expect none")); } } } - if parts.txindex_to_height.needs(height) { - wtx.insert( - parts.txindex_to_height.data(), - Slice::from(txindex), - Slice::from(height), - ); - } - Ok(()) }, )?; - if parts.height_to_last_addressindex.needs(height) { - wtx.insert( - parts.height_to_last_addressindex.data(), - Slice::from(height), - Slice::from(addressindex.decremented()), - ); - } + txindex_to_txid.into_iter().try_for_each(|(txindex, txid)| -> color_eyre::Result<()> { + vecdisks.txindex_to_txid.push_if_needed(txindex, txid)?; + vecdisks.txindex_to_height.push_if_needed(txindex, height)?; + Ok(()) + })?; + + vecdisks.height_to_last_txindex.push_if_needed(height, txindex.decremented())?; + vecdisks.height_to_last_txoutindex.push_if_needed(height, txoutindex.decremented())?; + vecdisks.height_to_last_addressindex.push_if_needed(height, addressindex.decremented())?; let should_snapshot = _height % 100 == 0 && !exit.active(); if should_snapshot { - export(&keyspace, wtx, &parts, height)?; + export(&keyspace, wtx, &parts, &mut vecdisks, height)?; wtx_opt.replace(keyspace.write_tx()); } else { wtx_opt.replace(wtx); } - txindex += txlen; + txindex += Txindex::from(tx_len); + txoutindex += Txoutindex::from(outputs_len); Ok(()) })?; dbg!(i.elapsed()); + pause(); + let wtx = wtx_opt.take().context("option should have wtx")?; - export(&keyspace, wtx, &parts, height)?; + export(&keyspace, wtx, &parts, &mut vecdisks, height)?; + + pause(); dbg!(i.elapsed()); Ok(()) } + +fn pause() { + let mut stdin = std::io::stdin(); + let mut stdout = std::io::stdout(); + + // We want the cursor to stay at the end of the line, so we print without a newline and flush manually. + write!(stdout, "Press any key to continue...").unwrap(); + stdout.flush().unwrap(); + + // Read a single byte and discard + let _ = stdin.read(&mut [0u8]).unwrap(); +} diff --git a/src/crates/bitbase/src/structs/addressbytes.rs b/src/crates/bitbase/src/structs/addressbytes.rs index d23e39beb..a7a3b773d 100644 --- a/src/crates/bitbase/src/structs/addressbytes.rs +++ b/src/crates/bitbase/src/structs/addressbytes.rs @@ -1,12 +1,19 @@ use biter::bitcoin::ScriptBuf; use color_eyre::eyre::eyre; use derive_deref::{Deref, DerefMut}; -use fjall::Slice; use super::Addresstype; -#[derive(Debug, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord)] -pub struct Addressbytes(Slice); +#[derive(Debug)] +pub enum Addressbytes { + P2PK65(P2PK65AddressBytes), + P2PK33(P2PK33AddressBytes), + P2PKH(P2PKHAddressBytes), + P2SH(P2SHAddressBytes), + P2WPKH(P2WPKHAddressBytes), + P2WSH(P2WSHAddressBytes), + P2TR(P2TRAddressBytes), +} impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes { type Error = color_eyre::Report; @@ -14,37 +21,47 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes { let (script, addresstype) = tuple; match addresstype { - Addresstype::P2PK => { + Addresstype::P2PK65 => { let bytes = script.as_bytes(); let bytes = match bytes.len() { 67 => &bytes[1..66], + _ => { + dbg!(bytes); + return Err(eyre!("Wrong len")); + } + }; + Ok(Self::P2PK65(P2PK65AddressBytes(U8x65::from(bytes)))) + } + Addresstype::P2PK33 => { + let bytes = script.as_bytes(); + let bytes = match bytes.len() { 35 => &bytes[1..34], _ => { dbg!(bytes); return Err(eyre!("Wrong len")); } }; - Ok(Self(bytes.into())) + Ok(Self::P2PK33(P2PK33AddressBytes(U8x33::from(bytes)))) } Addresstype::P2PKH => { let bytes = &script.as_bytes()[3..23]; - Ok(Self(bytes.into())) + Ok(Self::P2PKH(P2PKHAddressBytes(U8x20::from(bytes)))) } Addresstype::P2SH => { let bytes = &script.as_bytes()[2..22]; - Ok(Self(bytes.into())) + Ok(Self::P2SH(P2SHAddressBytes(U8x20::from(bytes)))) } Addresstype::P2WPKH => { let bytes = &script.as_bytes()[2..]; - Ok(Self(bytes.into())) + Ok(Self::P2WPKH(P2WPKHAddressBytes(U8x20::from(bytes)))) } Addresstype::P2WSH => { let bytes = &script.as_bytes()[2..]; - Ok(Self(bytes.into())) + Ok(Self::P2WSH(P2WSHAddressBytes(U8x32::from(bytes)))) } Addresstype::P2TR => { let bytes = &script.as_bytes()[2..]; - Ok(Self(bytes.into())) + Ok(Self::P2TR(P2TRAddressBytes(U8x32::from(bytes)))) } Addresstype::Multisig => Err(eyre!("multisig address type")), Addresstype::PushOnly => Err(eyre!("push_only address type")), @@ -55,13 +72,73 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes { } } -impl From for Addressbytes { - fn from(value: Slice) -> Self { - Self(value) +#[derive(Debug, Clone, Deref)] +pub struct P2PK65AddressBytes(U8x65); + +#[derive(Debug, Clone, Deref)] +pub struct P2PK33AddressBytes(U8x33); + +#[derive(Debug, Clone, Deref)] +pub struct P2PKHAddressBytes(U8x20); + +#[derive(Debug, Clone, Deref)] +pub struct P2SHAddressBytes(U8x20); + +#[derive(Debug, Clone, Deref)] +pub struct P2WPKHAddressBytes(U8x20); + +#[derive(Debug, Clone, Deref)] +pub struct P2WSHAddressBytes(U8x32); + +#[derive(Debug, Clone, Deref)] +pub struct P2TRAddressBytes(U8x32); + +#[derive(Debug, Clone, Deref, DerefMut)] +pub struct U8x20([u8; 20]); +impl From<&[u8]> for U8x20 { + fn from(slice: &[u8]) -> Self { + let mut arr = [0; 20]; + arr.copy_from_slice(slice); + Self(arr) } } -impl From<&Addressbytes> for Slice { - fn from(value: &Addressbytes) -> Self { - value.0.clone() + +#[derive(Debug, Clone, Deref, DerefMut)] +pub struct U8x32([u8; 32]); +impl From<&[u8]> for U8x32 { + fn from(slice: &[u8]) -> Self { + let mut arr = [0; 32]; + arr.copy_from_slice(slice); + Self(arr) + } +} + +#[derive(Debug, Clone, Deref, DerefMut)] +pub struct U8x33([u8; 33]); +impl From<&[u8]> for U8x33 { + fn from(slice: &[u8]) -> Self { + let mut arr = [0; 33]; + arr.copy_from_slice(slice); + Self(arr) + } +} + +#[derive(Debug, Clone, Deref, DerefMut)] +pub struct U8x64([u8; 64]); +impl From<&[u8]> for U8x64 { + fn from(slice: &[u8]) -> Self { + let mut arr = [0; 64]; + arr.copy_from_slice(slice); + Self(arr) + } +} + +#[derive(Debug, Clone, Deref, DerefMut)] +pub struct U8x65([u8; 65]); +impl From<&[u8]> for U8x65 { + fn from(slice: &[u8]) -> Self { + let mut arr = [0; 65]; + arr.copy_from_slice(slice); + Self(arr) } } diff --git a/src/crates/bitbase/src/structs/addressindex.rs b/src/crates/bitbase/src/structs/addressindex.rs index 87fca33b2..a5d2c9774 100644 --- a/src/crates/bitbase/src/structs/addressindex.rs +++ b/src/crates/bitbase/src/structs/addressindex.rs @@ -39,9 +39,27 @@ impl From for u64 { } } -impl From for Addressindex { - fn from(slice: Slice) -> Self { - Self(slice.read_u32()) +impl From for Addressindex { + fn from(value: usize) -> Self { + Self(value as u32) + } +} +impl From for usize { + fn from(value: Addressindex) -> Self { + value.0 as usize + } +} + +impl TryFrom for Addressindex { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + Self::try_from(&value[..]) + } +} +impl TryFrom<&[u8]> for Addressindex { + type Error = color_eyre::Report; + fn try_from(value: &[u8]) -> Result { + Ok(Self::from(value.read_be_u32()?)) } } impl From for Slice { diff --git a/src/crates/bitbase/src/structs/addressindextxoutindex.rs b/src/crates/bitbase/src/structs/addressindextxoutindex.rs new file mode 100644 index 000000000..cc89164b1 --- /dev/null +++ b/src/crates/bitbase/src/structs/addressindextxoutindex.rs @@ -0,0 +1,38 @@ +use fjall::Slice; + +use super::{Addressindex, Txoutindex}; + +#[derive(Debug)] +pub struct Addressindextxoutindex { + addressindex: Addressindex, + txoutindex: Txoutindex, +} + +impl From<(Addressindex, Txoutindex)> for Addressindextxoutindex { + fn from(value: (Addressindex, Txoutindex)) -> Self { + Self { + addressindex: value.0, + txoutindex: value.1, + } + } +} + +impl From for Slice { + fn from(value: Addressindextxoutindex) -> Self { + let addressindex_slice = Self::from(value.addressindex); + let txindexvout_slice = Self::from(value.txoutindex); + Self::from([addressindex_slice, txindexvout_slice].concat()) + } +} +impl TryFrom for Addressindextxoutindex { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + let addressindex = Addressindex::try_from(&value[..Addressindex::BYTES])?; + let txindexvout = Txoutindex::try_from(&value[Addressindex::BYTES..])?; + + Ok(Self { + addressindex, + txoutindex: txindexvout, + }) + } +} diff --git a/src/crates/bitbase/src/structs/addresstxoutindex.rs b/src/crates/bitbase/src/structs/addresstxoutindex.rs deleted file mode 100644 index 02a5d0d82..000000000 --- a/src/crates/bitbase/src/structs/addresstxoutindex.rs +++ /dev/null @@ -1,35 +0,0 @@ -use fjall::Slice; - -use super::{Addressindex, Txoutindex}; - -pub struct Addresstxoutindex { - addressindex: Addressindex, - txoutindex: Txoutindex, -} - -impl From<(Addressindex, Txoutindex)> for Addresstxoutindex { - fn from(value: (Addressindex, Txoutindex)) -> Self { - Self { - addressindex: value.0, - txoutindex: value.1, - } - } -} - -impl From for Slice { - fn from(value: Addresstxoutindex) -> Self { - let addressindex_slice = Self::from(value.addressindex); - let txoutindex_slice = Self::from(value.txoutindex); - Self::from([addressindex_slice, txoutindex_slice].concat()) - } -} -impl From for Addresstxoutindex { - fn from(value: Slice) -> Self { - let addressindex = Addressindex::from(Slice::from(&value[..Addressindex::BYTES])); - let txoutindex = Txoutindex::from(Slice::from(&value[Addressindex::BYTES..])); - Self { - addressindex, - txoutindex, - } - } -} diff --git a/src/crates/bitbase/src/structs/addresstype.rs b/src/crates/bitbase/src/structs/addresstype.rs index ff13060ac..abc37a296 100644 --- a/src/crates/bitbase/src/structs/addresstype.rs +++ b/src/crates/bitbase/src/structs/addresstype.rs @@ -1,12 +1,9 @@ use biter::bitcoin::ScriptBuf; -use color_eyre::eyre::eyre; -use fjall::Slice; - -use super::SliceExtended; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Addresstype { - P2PK, + P2PK65, + P2PK33, P2PKH, P2SH, P2WPKH, @@ -22,7 +19,16 @@ pub enum Addresstype { impl From<&ScriptBuf> for Addresstype { fn from(script: &ScriptBuf) -> Self { if script.is_p2pk() { - Self::P2PK + let bytes = script.as_bytes(); + + match bytes.len() { + 67 => Self::P2PK65, + 35 => Self::P2PK33, + _ => { + dbg!(bytes); + unreachable!() + } + } } else if script.is_p2pkh() { Self::P2PKH } else if script.is_p2sh() { @@ -46,19 +52,3 @@ impl From<&ScriptBuf> for Addresstype { } } } - -impl TryFrom for Addresstype { - type Error = color_eyre::Report; - fn try_from(value: Slice) -> Result { - match value.read_u8() { - x if x == Addresstype::P2PK as u8 => Ok(Addresstype::P2PK), - x if x == Addresstype::P2PKH as u8 => Ok(Addresstype::P2PKH), - _ => Err(eyre!("Unknown type")), - } - } -} -impl From for Slice { - fn from(addresstype: Addresstype) -> Self { - (addresstype as u8).to_be_bytes().into() - } -} diff --git a/src/crates/bitbase/src/structs/addresstypeindex.rs b/src/crates/bitbase/src/structs/addresstypeindex.rs new file mode 100644 index 000000000..15c2f4ba3 --- /dev/null +++ b/src/crates/bitbase/src/structs/addresstypeindex.rs @@ -0,0 +1,81 @@ +use derive_deref::{Deref, DerefMut}; +use fjall::Slice; + +use super::SliceExtended; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +pub struct Addresstypeindex(u32); + +impl Addresstypeindex { + pub fn decremented(self) -> Self { + Self(*self - 1) + } + + pub fn increment(&mut self) { + self.0 += 1; + } + + pub fn incremented(self) -> Self { + Self(*self + 1) + } +} + +impl From for Addresstypeindex { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for Addresstypeindex { + fn from(value: u64) -> Self { + Self(value as u32) + } +} +impl From for u64 { + fn from(value: Addresstypeindex) -> Self { + value.0 as u64 + } +} + +impl From for Addresstypeindex { + fn from(value: usize) -> Self { + Self(value as u32) + } +} +impl From for usize { + fn from(value: Addresstypeindex) -> Self { + value.0 as usize + } +} + +impl TryFrom for Addresstypeindex { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + Self::try_from(&value[..]) + } +} +impl TryFrom<&[u8]> for Addresstypeindex { + type Error = color_eyre::Report; + fn try_from(value: &[u8]) -> Result { + Ok(Self::from(value.read_be_u32()?)) + } +} +impl From for Slice { + fn from(value: Addresstypeindex) -> Self { + value.to_be_bytes().into() + } +} + +// impl Bytes for Addresstypeindex { +// const SIZE: usize = size_of::(); + +// type ByteArray = [u8; Self::SIZE]; + +// // fn try_from_bytes(bytes: &[u8]) -> color_eyre::Result { +// // Ok(Self(Self::read_u32(bytes))) +// // } + +// fn to_bytes(&self) -> Self::ByteArray { +// self.to_ne_bytes() +// } +// } diff --git a/src/crates/bitbase/src/structs/height.rs b/src/crates/bitbase/src/structs/height.rs index cd5aa23fb..3f8f0f346 100644 --- a/src/crates/bitbase/src/structs/height.rs +++ b/src/crates/bitbase/src/structs/height.rs @@ -12,41 +12,6 @@ use super::SliceExtended; #[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct Height(u32); -impl From for Height { - fn from(slice: Slice) -> Self { - Self(slice.read_u32()) - } -} -impl From for Slice { - fn from(value: Height) -> Self { - value.to_be_bytes().into() - } -} - -impl From for Height { - fn from(value: u32) -> Self { - Self(value) - } -} - -impl From for Height { - fn from(value: usize) -> Self { - Self(value as u32) - } -} -impl From for usize { - fn from(value: Height) -> Self { - value.0 as usize - } -} - -impl TryFrom<&bitcoincore_rpc::Client> for Height { - type Error = bitcoincore_rpc::Error; - fn try_from(value: &bitcoincore_rpc::Client) -> Result { - Ok((value.get_blockchain_info()?.blocks as usize - 1).into()) - } -} - impl PartialEq for Height { fn eq(&self, other: &u64) -> bool { **self == *other as u32 @@ -109,3 +74,53 @@ impl fmt::Display for Height { write!(f, "{}", **self) } } + +impl TryFrom for Height { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + Ok(Self::from((&value[..]).read_be_u32()?)) + } +} +impl From for Slice { + fn from(value: Height) -> Self { + value.to_be_bytes().into() + } +} + +impl From for Height { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for Height { + fn from(value: usize) -> Self { + Self(value as u32) + } +} +impl From for usize { + fn from(value: Height) -> Self { + value.0 as usize + } +} + +impl TryFrom<&bitcoincore_rpc::Client> for Height { + type Error = bitcoincore_rpc::Error; + fn try_from(value: &bitcoincore_rpc::Client) -> Result { + Ok((value.get_blockchain_info()?.blocks as usize - 1).into()) + } +} + +// impl Bytes for Height { +// const SIZE: usize = size_of::(); + +// type ByteArray = [u8; Self::SIZE]; + +// // fn try_from_bytes(bytes: &[u8]) -> color_eyre::Result { +// // Ok(Self(Self::read_u32(bytes))) +// // } + +// fn to_bytes(&self) -> Self::ByteArray { +// self.to_ne_bytes() +// } +// } diff --git a/src/crates/bitbase/src/structs/mod.rs b/src/crates/bitbase/src/structs/mod.rs index 6b557004e..123921c06 100644 --- a/src/crates/bitbase/src/structs/mod.rs +++ b/src/crates/bitbase/src/structs/mod.rs @@ -1,7 +1,8 @@ mod addressbytes; mod addressindex; -mod addresstxoutindex; +mod addressindextxoutindex; mod addresstype; +mod addresstypeindex; mod amount; mod exit; mod height; @@ -10,13 +11,17 @@ mod partitions; mod prefix; mod slice; mod txindex; +mod txindexvout; mod txoutindex; +mod vecdisk; +mod vecdisks; mod version; pub use addressbytes::*; pub use addressindex::*; -pub use addresstxoutindex::*; +pub use addressindextxoutindex::*; pub use addresstype::*; +pub use addresstypeindex::*; pub use amount::*; pub use exit::*; pub use height::*; @@ -25,5 +30,8 @@ pub use partitions::*; pub use prefix::*; pub use slice::*; pub use txindex::*; +pub use txindexvout::*; pub use txoutindex::*; +pub use vecdisk::*; +pub use vecdisks::*; pub use version::*; diff --git a/src/crates/bitbase/src/structs/partition.rs b/src/crates/bitbase/src/structs/partition.rs index 4f184442f..23791188d 100644 --- a/src/crates/bitbase/src/structs/partition.rs +++ b/src/crates/bitbase/src/structs/partition.rs @@ -15,38 +15,52 @@ impl Partition { pub const VERSION: &str = "version"; pub const HEIGHT: &str = "height"; - pub fn import(keyspace: &TransactionalKeyspace, name: &str, version: Version, exit: &Exit) -> Result { + pub fn import( + keyspace: &TransactionalKeyspace, + name: &str, + version: Version, + exit: &Exit, + ) -> color_eyre::Result { let data = Self::open_data(keyspace, name)?; let meta = Self::open_meta(keyspace, name)?; + let mut height = None; + if let Some(height_res) = meta.get(Self::HEIGHT)?.map(Height::try_from) { + height = Some(height_res?); + } + let mut this = Self { version, - height: meta.get(Self::HEIGHT)?.map(Height::from), + height, data, meta, }; + let mut different_version = false; if let Some(slice) = this.meta.get(Self::VERSION)? { - if version != Version::from(slice) { - this = this.reset(keyspace, name, exit)?; - } + different_version = Version::try_from(slice).map_or(true, |version2| version != version2); + } + + if different_version { + this = this.reset(keyspace, name, exit)?; } Ok(this) } fn open_data(keyspace: &TransactionalKeyspace, name: &str) -> Result { - keyspace.open_partition(&format!("{name}-data"), Self::create_options()) + keyspace.open_partition(&format!("{name}_data"), Self::create_options()) } fn open_meta(keyspace: &TransactionalKeyspace, name: &str) -> Result { - keyspace.open_partition(&format!("{name}-meta"), Self::create_options()) + keyspace.open_partition(&format!("{name}_meta"), Self::create_options()) } fn create_options() -> PartitionCreateOptions { PartitionCreateOptions::default().manual_journal_persist(true) } + // TODO: Still needed ? pub fn is_safe(&self, height: Height) -> bool { self.height.is_some_and(|self_height| self_height >= height) } diff --git a/src/crates/bitbase/src/structs/partitions.rs b/src/crates/bitbase/src/structs/partitions.rs index 1b08c8c1e..2563d468f 100644 --- a/src/crates/bitbase/src/structs/partitions.rs +++ b/src/crates/bitbase/src/structs/partitions.rs @@ -6,22 +6,11 @@ use crate::structs::{Exit, Height, Partition, Version}; pub struct Partitions { pub addressbytes_prefix_to_addressindex: Partition, - pub addressindex_to_addressbytes: Partition, - pub addressindex_to_addresstype: Partition, - pub addresstxoutindexes_in: Partition, - pub addresstxoutindexes_out: Partition, + pub addressindextxoutindex_in: Partition, + pub addressindextxoutindex_out: Partition, pub blockhash_prefix_to_height: Partition, - pub height_to_blockhash: Partition, - pub height_to_first_addressindex: Partition, - pub height_to_first_txindex: Partition, - pub height_to_last_addressindex: Partition, - pub height_to_last_txindex: Partition, pub txid_prefix_to_txindex: Partition, - pub txindex_to_height: Partition, - pub txindex_to_txid: Partition, - pub txoutindex_to_addressindex: Partition, - pub txoutindex_to_amount: Partition, - pub zero_txoutindexes: Partition, + pub txindexvout_to_txoutindex: Partition, } const UNSAFE_BLOCKS: usize = 100; @@ -35,52 +24,21 @@ impl Partitions { Version::from(1), exit, )?, - addressindex_to_addressbytes: Partition::import( - keyspace, - "addressindex_to_addressbytes", - Version::from(1), - exit, - )?, - addressindex_to_addresstype: Partition::import( - keyspace, - "addressindex_to_addresstype", - Version::from(1), - exit, - )?, - addresstxoutindexes_in: Partition::import(keyspace, "addresstxoutindexes_in", Version::from(1), exit)?, - addresstxoutindexes_out: Partition::import(keyspace, "addresstxoutindexes_out", Version::from(1), exit)?, + addressindextxoutindex_in: Partition::import(keyspace, "addresstxoutindexes_in", Version::from(1), exit)?, + addressindextxoutindex_out: Partition::import(keyspace, "addresstxoutindexes_out", Version::from(1), exit)?, blockhash_prefix_to_height: Partition::import( keyspace, "blockhash_prefix_to_height", Version::from(1), exit, )?, - height_to_blockhash: Partition::import(keyspace, "height_to_blockhash", Version::from(1), exit)?, - height_to_first_addressindex: Partition::import( - keyspace, - "height_to_first_addressindex", - Version::from(1), - exit, - )?, - height_to_first_txindex: Partition::import(keyspace, "height_to_first_txindex", Version::from(1), exit)?, - height_to_last_addressindex: Partition::import( - keyspace, - "height_to_last_addressindex", - Version::from(1), - exit, - )?, - height_to_last_txindex: Partition::import(keyspace, "height_to_last_txindex", Version::from(1), exit)?, txid_prefix_to_txindex: Partition::import(keyspace, "txid_prefix_to_txindex", Version::from(1), exit)?, - txindex_to_height: Partition::import(keyspace, "txindex_to_height", Version::from(1), exit)?, - txindex_to_txid: Partition::import(keyspace, "txindex_to_txid", Version::from(1), exit)?, - txoutindex_to_addressindex: Partition::import( + txindexvout_to_txoutindex: Partition::import( keyspace, - "txoutindex_to_addressindex", + "txindexvout_to_txoutindex", Version::from(1), exit, )?, - txoutindex_to_amount: Partition::import(keyspace, "txoutindex_to_amount", Version::from(1), exit)?, - zero_txoutindexes: Partition::import(keyspace, "zero_txoutindexes", Version::from(1), exit)?, }) } @@ -221,6 +179,7 @@ impl Partitions { // todo!("clear addresstxoutindexes_out") // todo!("clear addresstxoutindexes_in") // todo!("clear zero_txoutindexes") + // todo!("clear txindexvout_to_txoutindex") // Ok(()) } @@ -228,22 +187,11 @@ impl Partitions { fn to_vec(&self) -> Vec<&Partition> { vec![ &self.addressbytes_prefix_to_addressindex, - &self.addressindex_to_addressbytes, - &self.addressindex_to_addresstype, - &self.addresstxoutindexes_in, - &self.addresstxoutindexes_out, + &self.addressindextxoutindex_in, + &self.addressindextxoutindex_out, &self.blockhash_prefix_to_height, - &self.height_to_blockhash, - &self.height_to_first_addressindex, - &self.height_to_first_txindex, - &self.height_to_last_addressindex, - &self.height_to_last_txindex, &self.txid_prefix_to_txindex, - &self.txindex_to_height, - &self.txindex_to_txid, - &self.txoutindex_to_addressindex, - &self.txoutindex_to_amount, - &self.zero_txoutindexes, + &self.txindexvout_to_txoutindex, ] } } diff --git a/src/crates/bitbase/src/structs/prefix.rs b/src/crates/bitbase/src/structs/prefix.rs index 951979107..661a3971c 100644 --- a/src/crates/bitbase/src/structs/prefix.rs +++ b/src/crates/bitbase/src/structs/prefix.rs @@ -1,25 +1,55 @@ use biter::bitcoin::{BlockHash, Txid}; +use derive_deref::Deref; +use fjall::Slice; use super::Addressbytes; -pub trait Prefix { - fn prefix(&self) -> &[u8]; +#[derive(Debug, Deref, PartialEq, Eq, PartialOrd, Ord)] +pub struct Prefix(Slice); +impl From<&[u8]> for Prefix { + fn from(value: &[u8]) -> Self { + Self(Slice::from(&value[..8])) + } } +// pub struct Prefix([u8; 8]); +// impl From<&[u8]> for Prefix { +// fn from(value: &[u8]) -> Self { +// let mut buf: [u8; 8] = [0; 8]; +// value.iter().take(8).enumerate().for_each(|(i, v)| { +// buf[i] = *v; +// }); +// Self(buf) +// } +// } -impl Prefix for Addressbytes { - fn prefix(&self) -> &[u8] { - &self[..8] +#[derive(Debug, Deref)] +pub struct AddressbytesPrefix(Prefix); +impl From<&Addressbytes> for AddressbytesPrefix { + fn from(value: &Addressbytes) -> Self { + Self(Prefix::from(match value { + Addressbytes::P2PK65(bytes) => &bytes[..], + Addressbytes::P2PK33(bytes) => &bytes[..], + Addressbytes::P2PKH(bytes) => &bytes[..], + Addressbytes::P2SH(bytes) => &bytes[..], + Addressbytes::P2WPKH(bytes) => &bytes[..], + Addressbytes::P2WSH(bytes) => &bytes[..], + Addressbytes::P2TR(bytes) => &bytes[..], + })) } } -impl Prefix for BlockHash { - fn prefix(&self) -> &[u8] { - &self[..8] +#[derive(Debug, Deref)] +pub struct BlockHashPrefix(Prefix); +impl From<&BlockHash> for BlockHashPrefix { + fn from(value: &BlockHash) -> Self { + Self(Prefix::from(&value[..])) } } -impl Prefix for Txid { - fn prefix(&self) -> &[u8] { - &self[..8] +#[derive(Debug, Deref, PartialEq, Eq, PartialOrd, Ord)] +pub struct TxidPrefix(Prefix); +impl From<&Txid> for TxidPrefix { + fn from(value: &Txid) -> Self { + Self(Prefix::from(&value[..])) } } diff --git a/src/crates/bitbase/src/structs/slice.rs b/src/crates/bitbase/src/structs/slice.rs index 98e47c7f3..4ded832b1 100644 --- a/src/crates/bitbase/src/structs/slice.rs +++ b/src/crates/bitbase/src/structs/slice.rs @@ -1,49 +1,48 @@ -use std::io::Read; - -use fjall::Slice; +use color_eyre::eyre::eyre; #[allow(unused)] pub trait SliceExtended { - fn default() -> Self; - fn read_u8(&self) -> u8; - fn read_u16(&self) -> u16; - fn read_u32(&self) -> u32; - fn read_u64(&self) -> u64; - fn read_exact(&self, buf: &mut [u8]); + fn read_be_u8(&self) -> color_eyre::Result; + fn read_be_u16(&self) -> color_eyre::Result; + fn read_be_u32(&self) -> color_eyre::Result; + fn read_be_u64(&self) -> color_eyre::Result; + fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()>; } -impl SliceExtended for Slice { - fn default() -> Self { - Self::new(&[]) - } - - fn read_u8(&self) -> u8 { +impl SliceExtended for &[u8] { + fn read_be_u8(&self) -> color_eyre::Result { let mut buf: [u8; 1] = [0; 1]; - self.read_exact(&mut buf); - u8::from_be_bytes(buf) + self.read_exact(&mut buf)?; + Ok(u8::from_be_bytes(buf)) } - fn read_u16(&self) -> u16 { + fn read_be_u16(&self) -> color_eyre::Result { let mut buf: [u8; 2] = [0; 2]; - self.read_exact(&mut buf); - u16::from_be_bytes(buf) + self.read_exact(&mut buf)?; + Ok(u16::from_be_bytes(buf)) } - fn read_u32(&self) -> u32 { + fn read_be_u32(&self) -> color_eyre::Result { let mut buf: [u8; 4] = [0; 4]; - self.read_exact(&mut buf); - u32::from_be_bytes(buf) + self.read_exact(&mut buf)?; + Ok(u32::from_be_bytes(buf)) } - fn read_u64(&self) -> u64 { + fn read_be_u64(&self) -> color_eyre::Result { let mut buf: [u8; 8] = [0; 8]; - self.read_exact(&mut buf); - u64::from_be_bytes(buf) + self.read_exact(&mut buf)?; + Ok(u64::from_be_bytes(buf)) } - fn read_exact(&self, buf: &mut [u8]) { - self.bytes().take(buf.len()).enumerate().for_each(|(i, r)| { - buf[i] = r.unwrap(); + fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()> { + let buf_len = buf.len(); + if self.len() != buf_len { + dbg!(self.len(), buf_len); + return Err(eyre!("Not exact len")); + } + self.iter().take(buf_len).enumerate().for_each(|(i, r)| { + buf[i] = *r; }); + Ok(()) } } diff --git a/src/crates/bitbase/src/structs/txindex.rs b/src/crates/bitbase/src/structs/txindex.rs index 556c352cd..47bcdea8f 100644 --- a/src/crates/bitbase/src/structs/txindex.rs +++ b/src/crates/bitbase/src/structs/txindex.rs @@ -9,11 +9,26 @@ use super::SliceExtended; pub struct Txindex(u32); impl Txindex { - pub const BYTES: usize = size_of::(); - pub fn incremented(self) -> Self { Self(*self + 1) } + + pub fn decremented(self) -> Self { + Self(*self - 1) + } +} + +impl Add for Txindex { + type Output = Self; + fn add(self, rhs: Txindex) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl AddAssign for Txindex { + fn add_assign(&mut self, rhs: Txindex) { + self.0 += rhs.0 + } } impl From for Txindex { @@ -33,9 +48,27 @@ impl From for u64 { } } -impl From for Txindex { - fn from(slice: Slice) -> Self { - Self(slice.read_u32()) +impl From for Txindex { + fn from(value: usize) -> Self { + Self(value as u32) + } +} +impl From for usize { + fn from(value: Txindex) -> Self { + value.0 as usize + } +} + +impl TryFrom for Txindex { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + Self::try_from(&value[..]) + } +} +impl TryFrom<&[u8]> for Txindex { + type Error = color_eyre::Report; + fn try_from(value: &[u8]) -> Result { + Ok(Self::from(value.read_be_u32()?)) } } impl From for Slice { @@ -43,16 +76,3 @@ impl From for Slice { value.to_be_bytes().into() } } - -impl Add for Txindex { - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - Self(self.0 + rhs as u32) - } -} - -impl AddAssign for Txindex { - fn add_assign(&mut self, rhs: usize) { - self.0 += rhs as u32 - } -} diff --git a/src/crates/bitbase/src/structs/txindexvout.rs b/src/crates/bitbase/src/structs/txindexvout.rs new file mode 100644 index 000000000..0a75a529c --- /dev/null +++ b/src/crates/bitbase/src/structs/txindexvout.rs @@ -0,0 +1,51 @@ +use fjall::Slice; + +use super::{SliceExtended, Txindex}; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)] +pub struct Txindexvout { + pub txindex: Txindex, + pub vout: u32, +} + +const BYTES: usize = size_of::(); + +impl From for Txindexvout { + fn from(value: Txindex) -> Self { + Self { + txindex: value, + vout: 0, + } + } +} + +impl From<(Txindex, u32)> for Txindexvout { + fn from(value: (Txindex, u32)) -> Self { + Self { + txindex: value.0, + vout: value.1, + } + } +} + +impl From for Slice { + fn from(value: Txindexvout) -> Self { + let txindex_slice = Self::from(value.txindex); + let vout_slice = Self::from(value.vout.to_be_bytes()); + Self::from([txindex_slice, vout_slice].concat()) + } +} +impl TryFrom for Txindexvout { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + Self::try_from(&value[..]) + } +} +impl TryFrom<&[u8]> for Txindexvout { + type Error = color_eyre::Report; + fn try_from(value: &[u8]) -> Result { + let txindex = Txindex::try_from(&value[..BYTES])?; + let vout = (&value[BYTES..]).read_be_u32()?; + Ok(Self { txindex, vout }) + } +} diff --git a/src/crates/bitbase/src/structs/txoutindex.rs b/src/crates/bitbase/src/structs/txoutindex.rs index 69b194aff..92e1bc88d 100644 --- a/src/crates/bitbase/src/structs/txoutindex.rs +++ b/src/crates/bitbase/src/structs/txoutindex.rs @@ -1,46 +1,72 @@ +use std::ops::{Add, AddAssign}; + +use derive_deref::{Deref, DerefMut}; use fjall::Slice; -use super::{SliceExtended, Txindex}; +use super::SliceExtended; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)] -pub struct Txoutindex { - pub txindex: Txindex, - pub vout: u16, -} +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +pub struct Txoutindex(u64); impl Txoutindex { - pub const BYTES: usize = size_of::(); -} + pub fn incremented(self) -> Self { + Self(*self + 1) + } -impl From for Txoutindex { - fn from(value: Txindex) -> Self { - Self { - txindex: value, - vout: 0, - } + pub fn decremented(self) -> Self { + Self(*self - 1) } } -impl From<(Txindex, u16)> for Txoutindex { - fn from(value: (Txindex, u16)) -> Self { - Self { - txindex: value.0, - vout: value.1, - } +impl Add for Txoutindex { + type Output = Self; + fn add(self, rhs: Txoutindex) -> Self::Output { + Self(self.0 + rhs.0) } } +impl AddAssign for Txoutindex { + fn add_assign(&mut self, rhs: Txoutindex) { + self.0 += rhs.0 + } +} + +impl From for Txoutindex { + fn from(value: u64) -> Self { + Self(value) + } +} +impl From for u64 { + fn from(value: Txoutindex) -> Self { + value.0 + } +} + +impl From for Txoutindex { + fn from(value: usize) -> Self { + Self(value as u64) + } +} +impl From for usize { + fn from(value: Txoutindex) -> Self { + value.0 as usize + } +} + +impl TryFrom for Txoutindex { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + Self::try_from(&value[..]) + } +} +impl TryFrom<&[u8]> for Txoutindex { + type Error = color_eyre::Report; + fn try_from(value: &[u8]) -> Result { + Ok(Self::from(value.read_be_u64()?)) + } +} impl From for Slice { fn from(value: Txoutindex) -> Self { - let txindex_slice = Self::from(value.txindex); - let vout_slice = Self::from(value.vout.to_be_bytes()); - Self::from([txindex_slice, vout_slice].concat()) - } -} -impl From for Txoutindex { - fn from(value: Slice) -> Self { - let txindex = Txindex::from(Slice::from(&value[..Txindex::BYTES])); - let vout = Slice::from(&value[Txindex::BYTES..]).read_u16(); - Self { txindex, vout } + value.to_be_bytes().into() } } diff --git a/src/crates/bitbase/src/structs/vecdisk.rs b/src/crates/bitbase/src/structs/vecdisk.rs new file mode 100644 index 000000000..13aa4e9fe --- /dev/null +++ b/src/crates/bitbase/src/structs/vecdisk.rs @@ -0,0 +1,219 @@ +use std::{ + cmp::Ordering, + fmt::Debug, + fs::{File, OpenOptions}, + io::{self, Write}, + marker::PhantomData, + mem, + ops::Range, + path::Path, + sync::OnceLock, +}; + +use color_eyre::eyre::{eyre, ContextCompat}; + +use derive_deref::{Deref, DerefMut}; +use memmap2::{Mmap, MmapOptions}; + +/// +/// A Push only vec stored on disk using Mmap +/// +/// Reads (imports of Mmap) are lazy +/// +/// Stores only raw data without any overhead, and doesn't even have a header (TODO: which it should, at least to Err if wrong endian) +/// +/// The file isn't portable for speed reasons (TODO: but could be ?) +/// +#[derive(Debug)] +pub struct Vecdisk { + file: File, + mmaps: VecLazyMmap, + disk_len: usize, + cache: Vec, + phantom: PhantomData, +} + +/// In bytes +const MAX_PAGE_SIZE: usize = 4096; + +impl Vecdisk +where + I: Into, + T: Sized + Debug, +{ + pub const SIZE: usize = size_of::(); + + pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE; + /// In bytes + pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE; + + pub fn import(path: &Path) -> color_eyre::Result { + let file = Self::open_file(path)?; + + let mut this = Self { + disk_len: Self::byte_index_to_index(file.metadata()?.len() as usize), + file, + mmaps: VecLazyMmap::default(), + cache: vec![], + phantom: PhantomData, + }; + + this.reset_mmaps(); + + Ok(this) + } + + fn reset_mmaps(&mut self) { + self.mmaps + .reset((self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize); + } + + fn open_file(path: &Path) -> Result { + OpenOptions::new() + .read(true) + .create(true) + .truncate(false) + .append(true) + .open(path) + } + + #[inline] + fn index_to_mmap_index(index: usize) -> usize { + Self::index_to_byte_index(index) / Self::PAGE_SIZE + } + + #[inline] + fn index_to_range(index: usize) -> Range { + let index = Self::index_to_byte_index(index) % Self::PAGE_SIZE; + index..(index + Self::SIZE) + } + + #[inline] + fn index_to_byte_index(index: usize) -> usize { + index * Self::SIZE + } + + #[inline] + fn byte_index_to_index(byte_index: usize) -> usize { + byte_index / Self::SIZE + } + + #[allow(unused)] + #[inline] + pub fn get(&self, index: I) -> color_eyre::Result> { + self._get(index.into()) + } + fn _get(&self, index: usize) -> color_eyre::Result> { + if self.disk_len == 0 { + Ok(None) + } else if index > self.disk_len - 1 { + Ok(self.cache.get(index - self.disk_len)) + } else { + let mmap_index = Self::index_to_mmap_index(index); + + let mmap = self + .mmaps + .get(mmap_index) + .context("Expect mmap to be open")? + .get_or_load(MAX_PAGE_SIZE, (mmap_index * Self::PAGE_SIZE) as u64, &self.file); + + let range = Self::index_to_range(index); + let src = &mmap[range]; + + let (prefix, shorts, suffix) = unsafe { src.align_to::() }; + + if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() { + dbg!(&src, &prefix, &shorts, &suffix); + return Err(eyre!("align_to issue")); + } + + Ok(Some(&shorts[0])) + } + } + + #[allow(unused)] + pub fn first(&self) -> color_eyre::Result> { + self._get(0) + } + + #[allow(unused)] + pub fn last(&self) -> color_eyre::Result> { + let len = self.len(); + if len == 0 { + return Ok(None); + } + self._get(len - 1) + } + + pub fn push(&mut self, value: T) { + self.cache.push(value) + } + + pub fn push_if_needed(&mut self, index: I, value: T) -> color_eyre::Result<()> { + let len = self.len(); + let index = index.into(); + match len.cmp(&index) { + Ordering::Greater => Ok(()), + Ordering::Equal => { + self.push(value); + Ok(()) + } + Ordering::Less => { + dbg!(std::any::type_name::(), std::any::type_name::()); + dbg!(len, index, value); + Err(eyre!("Index is too high")) + } + } + } +} + +pub trait AnyVecdisk { + fn len(&self) -> usize; + fn flush(&mut self) -> color_eyre::Result<()>; +} + +impl AnyVecdisk for Vecdisk +where + I: Into, + T: Sized + Debug, +{ + fn len(&self) -> usize { + self.disk_len + self.cache.len() + } + + fn flush(&mut self) -> color_eyre::Result<()> { + self.disk_len += self.cache.len(); + self.reset_mmaps(); + + let mut bytes: Vec = vec![]; + + mem::take(&mut self.cache).into_iter().for_each(|v| { + let data: *const T = &v; + let data: *const u8 = data as *const u8; + let slice = unsafe { std::slice::from_raw_parts(data, Self::SIZE) }; + bytes.extend_from_slice(slice) + }); + + self.file.write_all(&bytes)?; + + Ok(()) + } +} + +#[derive(Debug, Default, Deref, DerefMut)] +struct VecLazyMmap(Vec); +impl VecLazyMmap { + pub fn reset(&mut self, len: usize) { + self.clear(); + self.resize_with(len, Default::default); + } +} + +/// Box to reduce the size, would be 24 instead +#[derive(Debug, Default, Deref)] +struct LazyMmap(OnceLock>); +impl LazyMmap { + pub fn get_or_load(&self, len: usize, offset: u64, file: &File) -> &Mmap { + self.get_or_init(|| Box::new(unsafe { MmapOptions::new().len(len).offset(offset).map(file).unwrap() })) + } +} diff --git a/src/crates/bitbase/src/structs/vecdisks.rs b/src/crates/bitbase/src/structs/vecdisks.rs new file mode 100644 index 000000000..37843c1ae --- /dev/null +++ b/src/crates/bitbase/src/structs/vecdisks.rs @@ -0,0 +1,248 @@ +use std::{fs, path::Path}; + +use biter::bitcoin::{BlockHash, Txid}; +use color_eyre::eyre::eyre; + +use super::{ + Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, AnyVecdisk, Exit, Height, P2PK33AddressBytes, + P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, + Txindex, Txoutindex, Vecdisk, +}; + +pub struct Vecdisks { + // TODO: + // + // Add + // txindex_to_fees + // height_to_fees + // height_to_utc_date + // height_to_timestamp + // + // NOT the following as because of reorg it's subjective + // date_to_fees + // date_to_first_height + // date_to_last_height + pub addressindex_to_addresstype: Vecdisk, + pub addressindex_to_addresstypeindex: Vecdisk, + pub height_to_blockhash: Vecdisk, + pub height_to_first_addressindex: Vecdisk, + pub height_to_first_txindex: Vecdisk, + pub height_to_first_txoutindex: Vecdisk, + pub height_to_last_addressindex: Vecdisk, + pub height_to_last_txindex: Vecdisk, + pub height_to_last_txoutindex: Vecdisk, + pub p2pk65index_to_p2pk65addressbytes: Vecdisk, + pub p2pk33index_to_p2pk33addressbytes: Vecdisk, + pub p2pkhindex_to_p2pkhaddressbytes: Vecdisk, + pub p2shindex_to_p2shaddressbytes: Vecdisk, + pub p2wpkhindex_to_p2wpkhaddressbytes: Vecdisk, + pub p2wshindex_to_p2wshaddressbytes: Vecdisk, + pub p2trindex_to_p2traddressbytes: Vecdisk, + pub txindex_to_height: Vecdisk, + pub txindex_to_txid: Vecdisk, + pub txoutindex_to_addressindex: Vecdisk, + pub txoutindex_to_amount: Vecdisk, +} + +// const UNSAFE_BLOCKS: usize = 100; + +impl Vecdisks { + pub fn import(path: &Path) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + addressindex_to_addresstype: Vecdisk::import(&path.join("addressindex_to_addresstype"))?, + addressindex_to_addresstypeindex: Vecdisk::import(&path.join("addressindex_to_addresstypeindex"))?, + height_to_blockhash: Vecdisk::import(&path.join("height_to_blockhash"))?, + height_to_first_addressindex: Vecdisk::import(&path.join("height_to_first_addressindex"))?, + height_to_first_txindex: Vecdisk::import(&path.join("height_to_first_txindex"))?, + height_to_first_txoutindex: Vecdisk::import(&path.join("height_to_first_txoutindex"))?, + height_to_last_addressindex: Vecdisk::import(&path.join("height_to_last_addressindex"))?, + height_to_last_txindex: Vecdisk::import(&path.join("height_to_last_txindex"))?, + height_to_last_txoutindex: Vecdisk::import(&path.join("height_to_last_txoutindex"))?, + p2pk65index_to_p2pk65addressbytes: Vecdisk::import(&path.join("p2pk65index_to_p2pk65addressbytes"))?, + p2pk33index_to_p2pk33addressbytes: Vecdisk::import(&path.join("p2pk33index_to_p2pk33addressbytes"))?, + p2pkhindex_to_p2pkhaddressbytes: Vecdisk::import(&path.join("p2pkhindex_to_p2pkhaddressbytes"))?, + p2shindex_to_p2shaddressbytes: Vecdisk::import(&path.join("p2shindex_to_p2shaddressbytes"))?, + p2wpkhindex_to_p2wpkhaddressbytes: Vecdisk::import(&path.join("p2wpkhindex_to_p2wpkhaddressbytes"))?, + p2wshindex_to_p2wshaddressbytes: Vecdisk::import(&path.join("p2wshindex_to_p2wshaddressbytes"))?, + p2trindex_to_p2traddressbytes: Vecdisk::import(&path.join("p2trindex_to_p2traddressbytes"))?, + txindex_to_height: Vecdisk::import(&path.join("txindex_to_height"))?, + txindex_to_txid: Vecdisk::import(&path.join("txindex_to_txid"))?, + txoutindex_to_addressindex: Vecdisk::import(&path.join("txoutindex_to_addressindex"))?, + txoutindex_to_amount: Vecdisk::import(&path.join("txoutindex_to_amount"))?, + }) + } + + pub fn addresstype_to_addressvecdisk(&self, addresstype: Addresstype) -> color_eyre::Result<&dyn AnyVecdisk> { + match addresstype { + Addresstype::P2PK65 => Ok(&self.p2pk65index_to_p2pk65addressbytes), + Addresstype::P2PK33 => Ok(&self.p2pk33index_to_p2pk33addressbytes), + Addresstype::P2PKH => Ok(&self.p2pkhindex_to_p2pkhaddressbytes), + Addresstype::P2SH => Ok(&self.p2shindex_to_p2shaddressbytes), + Addresstype::P2WPKH => Ok(&self.p2wpkhindex_to_p2wpkhaddressbytes), + Addresstype::P2WSH => Ok(&self.p2wshindex_to_p2wshaddressbytes), + Addresstype::P2TR => Ok(&self.p2trindex_to_p2traddressbytes), + _ => Err(eyre!("wrong address type")), + } + } + + pub fn push_addressbytes_if_needed( + &mut self, + index: Addresstypeindex, + addressbytes: Addressbytes, + ) -> color_eyre::Result<()> { + match addressbytes { + Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes), + Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes), + Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes), + Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes), + Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes), + Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes), + Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes), + } + } + + #[allow(unused)] + pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> { + panic!(); + // let mut txindex = None; + + // wtx.range(self.height_to_blockhash.data(), Slice::from(height)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (height_slice, slice_blockhash) = slice?; + // let blockhash = BlockHash::from_slice(&slice_blockhash)?; + + // wtx.remove(self.height_to_blockhash.data(), height_slice); + + // wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix()); + + // if txindex.is_none() { + // txindex.replace( + // wtx.get(self.height_to_first_txindex.data(), height_slice)? + // .context("for height to have first txindex")?, + // ); + // } + // wtx.remove(self.height_to_first_txindex.data(), height_slice); + // wtx.remove(self.height_to_last_txindex.data(), height_slice); + + // Ok(()) + // })?; + + // let txindex = txindex.context("txindex to not be none by now")?; + + // wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (slice_txindex, slice_txid) = slice?; + // let txindex = Txindex::from(slice_txindex); + // let txid = Txid::from_slice(&slice_txid)?; + + // wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex)); + // wtx.remove(self.txindex_to_height.data(), Slice::from(txindex)); + // wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix()); + + // Ok(()) + // })?; + + // let txoutindex = Txoutindex::from(txindex); + + // let mut addressindexes = BTreeSet::new(); + + // wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (txoutindex_slice, _) = slice?; + + // wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice); + + // if let Some(addressindex_slice) = + // wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)? + // { + // wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice); + + // let addressindex = Addressindex::from(addressindex_slice); + // addressindexes.insert(addressindex); + + // let txoutindex = Txoutindex::from(txoutindex_slice); + // let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex)); + + // wtx.remove( + // self.addressindex_to_txoutindexes.data(), + // Slice::from(addresstxoutindex), + // ); + // } + + // Ok(()) + // })?; + + // addressindexes + // .into_iter() + // .filter(|addressindex| { + // let is_empty = wtx + // .prefix( + // self.addressindex_to_txoutindexes.data(), + // Slice::from(*addressindex), + // ) + // .next() + // .is_none(); + // is_empty + // }) + // .try_for_each(|addressindex| -> color_eyre::Result<()> { + // let addressindex_slice = Slice::from(addressindex); + + // let addressbytes = Addressbytes::from( + // wtx.get( + // self.addressindex_to_addressbytes.data(), + // &addressindex_slice, + // )? + // .context("addressindex_to_address to have value")?, + // ); + // wtx.remove( + // self.addressbytes_prefix_to_addressindex.data(), + // addressbytes.prefix(), + // ); + // wtx.remove( + // self.addressindex_to_addressbytes.data(), + // &addressindex_slice, + // ); + // wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice); + + // Ok(()) + // })?; + // + + // todo!("clear addresstxoutindexes_out") + // todo!("clear addresstxoutindexes_in") + // todo!("clear zero_txoutindexes") + + // Ok(()) + } + + pub fn flush(&mut self) -> color_eyre::Result<()> { + self.as_mut_vec().into_iter().try_for_each(AnyVecdisk::flush) + } + + fn as_mut_vec(&mut self) -> Vec<&mut dyn AnyVecdisk> { + vec![ + &mut self.addressindex_to_addresstype, + &mut self.addressindex_to_addresstypeindex, + &mut self.height_to_blockhash, + &mut self.height_to_first_addressindex, + &mut self.height_to_first_txindex, + &mut self.height_to_first_txoutindex, + &mut self.height_to_last_addressindex, + &mut self.height_to_last_txindex, + &mut self.height_to_last_txoutindex, + &mut self.p2pk65index_to_p2pk65addressbytes, + &mut self.p2pk33index_to_p2pk33addressbytes, + &mut self.p2pkhindex_to_p2pkhaddressbytes, + &mut self.p2shindex_to_p2shaddressbytes, + &mut self.p2wpkhindex_to_p2wpkhaddressbytes, + &mut self.p2wshindex_to_p2wshaddressbytes, + &mut self.p2trindex_to_p2traddressbytes, + &mut self.txindex_to_height, + &mut self.txindex_to_txid, + &mut self.txoutindex_to_addressindex, + &mut self.txoutindex_to_amount, + ] + } +} diff --git a/src/crates/bitbase/src/structs/version.rs b/src/crates/bitbase/src/structs/version.rs index 80fdf235c..cc94051ed 100644 --- a/src/crates/bitbase/src/structs/version.rs +++ b/src/crates/bitbase/src/structs/version.rs @@ -12,12 +12,18 @@ impl From for Version { } } -impl From for Version { - fn from(slice: Slice) -> Self { - Self(slice.read_u8()) +impl TryFrom for Version { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + Self::try_from(&value[..]) + } +} +impl TryFrom<&[u8]> for Version { + type Error = color_eyre::Report; + fn try_from(value: &[u8]) -> Result { + Ok(Self::from(value.read_be_u8()?)) } } - impl From for Slice { fn from(value: Version) -> Self { value.to_be_bytes().into()