From bf31ee5fd64008cdae2352f32d97c24cecf23191 Mon Sep 17 00:00:00 2001 From: nym21 Date: Mon, 6 Jan 2025 10:45:04 +0100 Subject: [PATCH] bitbase: move to transactional --- .../addressbytes_prefix_to_addressindex.rs | 41 --- .../database/addressindex_to_addressbytes.rs | 43 --- .../database/addressindex_to_addresstype.rs | 38 --- .../database/addressindex_to_txoutindexes.rs | 42 --- .../database/blockhash_prefix_to_height.rs | 35 --- .../src/database/height_to_blockhash.rs | 37 --- .../bitbase/src/database/height_to_txindex.rs | 45 --- src/crates/bitbase/src/database/mod.rs | 256 ---------------- .../src/database/txid_prefix_to_txindex.rs | 46 --- .../bitbase/src/database/txindex_to_height.rs | 29 -- .../bitbase/src/database/txindex_to_txid.rs | 27 -- .../database/txoutindex_to_addressindex.rs | 30 -- .../src/database/txoutindex_to_amount.rs | 29 -- src/crates/bitbase/src/main.rs | 282 +++++++++++++----- .../bitbase/src/structs/addressbytes.rs | 76 +++-- .../bitbase/src/structs/addressindex.rs | 4 + src/crates/bitbase/src/structs/addresstype.rs | 42 ++- src/crates/bitbase/src/structs/fjall.rs | 143 --------- src/crates/bitbase/src/structs/mod.rs | 8 +- src/crates/bitbase/src/structs/partition.rs | 104 +++++++ src/crates/bitbase/src/structs/partitions.rs | 278 +++++++++++++++++ src/crates/bitbase/src/structs/prefix.rs | 25 ++ src/crates/bitbase/src/structs/version.rs | 6 + src/crates/biter/src/lib.rs | 2 +- 24 files changed, 714 insertions(+), 954 deletions(-) delete mode 100644 src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs delete mode 100644 src/crates/bitbase/src/database/addressindex_to_addressbytes.rs delete mode 100644 src/crates/bitbase/src/database/addressindex_to_addresstype.rs delete mode 100644 src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs delete mode 100644 src/crates/bitbase/src/database/blockhash_prefix_to_height.rs delete mode 100644 src/crates/bitbase/src/database/height_to_blockhash.rs delete mode 100644 src/crates/bitbase/src/database/height_to_txindex.rs delete mode 100644 src/crates/bitbase/src/database/mod.rs delete mode 100644 src/crates/bitbase/src/database/txid_prefix_to_txindex.rs delete mode 100644 src/crates/bitbase/src/database/txindex_to_height.rs delete mode 100644 src/crates/bitbase/src/database/txindex_to_txid.rs delete mode 100644 src/crates/bitbase/src/database/txoutindex_to_addressindex.rs delete mode 100644 src/crates/bitbase/src/database/txoutindex_to_amount.rs delete mode 100644 src/crates/bitbase/src/structs/fjall.rs create mode 100644 src/crates/bitbase/src/structs/partition.rs create mode 100644 src/crates/bitbase/src/structs/partitions.rs create mode 100644 src/crates/bitbase/src/structs/prefix.rs diff --git a/src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs b/src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs deleted file mode 100644 index a72cee62a..000000000 --- a/src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs +++ /dev/null @@ -1,41 +0,0 @@ -use color_eyre::eyre::eyre; -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Addressbytes, Addressindex, Database, DatabaseTrait, Height, Version}; - -#[derive(Deref, DerefMut)] -pub struct AddressbytesPrefixToAddressindex(Database); - -impl AddressbytesPrefixToAddressindex { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "address_prefix_to_addressindex", - Self::version(), - )?)) - } - - pub fn insert( - &mut self, - addressbytes: &Addressbytes, - addressindex: Addressindex, - height: Height, - ) -> color_eyre::Result<()> { - if let Some(_height) = - self.fetch_update(addressbytes.to_prefix_slice(), addressindex.into(), height)? - { - dbg!(addressbytes, addressindex); - return Err(eyre!("AddressPrefixToAddressindex: key collision")); - } - Ok(()) - } - - pub fn remove(&mut self, addressbytes: &Addressbytes) { - self.0.remove(addressbytes.to_prefix_slice()) - } -} - -impl DatabaseTrait for AddressbytesPrefixToAddressindex { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/addressindex_to_addressbytes.rs b/src/crates/bitbase/src/database/addressindex_to_addressbytes.rs deleted file mode 100644 index 228bc3965..000000000 --- a/src/crates/bitbase/src/database/addressindex_to_addressbytes.rs +++ /dev/null @@ -1,43 +0,0 @@ -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Addressbytes, Addressindex, Database, DatabaseTrait, Height, Version}; - -#[derive(Deref, DerefMut)] -pub struct AddressindexToAddressbytes(Database); - -impl AddressindexToAddressbytes { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "addressindex_to_addressbytes", - Self::version(), - )?)) - } - - pub fn get(&self, addressindex: Addressindex) -> color_eyre::Result> { - if let Some(address) = self.0.get(addressindex.into())?.map(Addressbytes::try_from) { - Ok(Some(address?)) - } else { - Ok(None) - } - } - - pub fn insert( - &mut self, - addressindex: Addressindex, - addressbytes: &Addressbytes, - height: Height, - ) { - self.0 - .insert(addressindex.into(), addressbytes.into(), height) - } - - pub fn remove(&mut self, addressindex: Addressindex) { - self.0.remove(addressindex.into()) - } -} - -impl DatabaseTrait for AddressindexToAddressbytes { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/addressindex_to_addresstype.rs b/src/crates/bitbase/src/database/addressindex_to_addresstype.rs deleted file mode 100644 index 32fc0046a..000000000 --- a/src/crates/bitbase/src/database/addressindex_to_addresstype.rs +++ /dev/null @@ -1,38 +0,0 @@ -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Addressindex, Addresstype, Database, DatabaseTrait, Height, Version}; - -#[derive(Deref, DerefMut)] -pub struct AddressindexToAddresstype(Database); - -impl AddressindexToAddresstype { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "addressindex_to_addresstype", - Self::version(), - )?)) - } - - // pub fn get(&self, addressindex: Addressindex) -> color_eyre::Result> { - // if let Some(addresstype) = self.0.get(addressindex.into())?.map(Addresstype::try_from) { - // Ok(Some(addresstype?)) - // } else { - // Ok(None) - // } - // } - - pub fn insert(&mut self, addressindex: Addressindex, addresstype: Addresstype, height: Height) { - self.0 - .insert(addressindex.into(), addresstype.into(), height) - } - - pub fn remove(&mut self, addressindex: Addressindex) { - self.0.remove(addressindex.into()) - } -} - -impl DatabaseTrait for AddressindexToAddresstype { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs b/src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs deleted file mode 100644 index 7f173c3a9..000000000 --- a/src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs +++ /dev/null @@ -1,42 +0,0 @@ -use derive_deref::{Deref, DerefMut}; -use fjall::Slice; - -use crate::structs::{ - Addressindex, Addresstxoutindex, Database, DatabaseTrait, Height, SliceExtended, Txoutindex, - Version, -}; - -#[derive(Deref, DerefMut)] -pub struct AddressindexToTxoutindexes(Database); - -impl AddressindexToTxoutindexes { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "addressindex_to_txoutindexes", - Self::version(), - )?)) - } - - pub fn insert(&mut self, addressindex: Addressindex, txoutindex: Txoutindex, height: Height) { - self.0.insert( - Addresstxoutindex::from((addressindex, txoutindex)).into(), - Slice::default(), - height, - ) - } - - pub fn remove(&mut self, addressindex: Addressindex, txoutindex: Txoutindex) { - self.0 - .remove(Addresstxoutindex::from((addressindex, txoutindex)).into()); - } - - pub fn is_empty(&self, addressindex: Addressindex) -> bool { - self.prefix(Slice::from(addressindex)).next().is_none() - } -} - -impl DatabaseTrait for AddressindexToTxoutindexes { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/blockhash_prefix_to_height.rs b/src/crates/bitbase/src/database/blockhash_prefix_to_height.rs deleted file mode 100644 index 40c83118b..000000000 --- a/src/crates/bitbase/src/database/blockhash_prefix_to_height.rs +++ /dev/null @@ -1,35 +0,0 @@ -use biter::bitcoin::BlockHash; -use color_eyre::eyre::eyre; -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Database, DatabaseTrait, Height, Version}; - -#[derive(Deref, DerefMut)] -pub struct BlockhashPrefixToHeight(Database); - -impl BlockhashPrefixToHeight { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "blockhash_suffix_to_height", - Self::version(), - )?)) - } - - pub fn insert(&mut self, blockhash: &BlockHash, height: Height) -> color_eyre::Result<()> { - if let Some(_height) = self.fetch_update(blockhash[..8].into(), height.into(), height)? { - // dbg!(height, Height::from(other), hash); - return Err(eyre!("BlockhashSuffixToHeight: key collision")); - } - Ok(()) - } - - pub fn remove(&mut self, blockhash: &BlockHash) { - self.0.remove((&blockhash[..]).into()) - } -} - -impl DatabaseTrait for BlockhashPrefixToHeight { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/height_to_blockhash.rs b/src/crates/bitbase/src/database/height_to_blockhash.rs deleted file mode 100644 index bce5882da..000000000 --- a/src/crates/bitbase/src/database/height_to_blockhash.rs +++ /dev/null @@ -1,37 +0,0 @@ -use biter::bitcoin::{hashes::Hash, BlockHash}; -use derive_deref::{Deref, DerefMut}; -use fjall::Slice; - -use crate::structs::{Database, DatabaseTrait, Height, Version}; - -#[derive(Deref, DerefMut)] -pub struct HeightToBlockhash(Database); - -impl HeightToBlockhash { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "height_to_blockhash", - Self::version(), - )?)) - } - - pub fn insert(&mut self, height: Height, blockhash: &BlockHash) { - self.0.insert(height.into(), blockhash[..].into(), height) - } - - pub fn get(&self, height: Height) -> fjall::Result> { - self.0 - .get(Slice::from(height)) - .map(|opt| opt.map(|slice| BlockHash::from_slice(&slice).unwrap())) - } - - pub fn remove(&mut self, height: Height) { - self.0.remove(Slice::from(height)) - } -} - -impl DatabaseTrait for HeightToBlockhash { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/height_to_txindex.rs b/src/crates/bitbase/src/database/height_to_txindex.rs deleted file mode 100644 index 0cdeb0781..000000000 --- a/src/crates/bitbase/src/database/height_to_txindex.rs +++ /dev/null @@ -1,45 +0,0 @@ -use derive_deref::{Deref, DerefMut}; -use fjall::Slice; - -use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version}; - -#[derive(Deref, DerefMut)] -pub struct HeightToTxindex(Database); - -#[derive(Debug, PartialEq, Eq)] -pub enum HeightToTxindexPosition { - First, - Last, -} - -impl HeightToTxindex { - pub fn import(position: HeightToTxindexPosition) -> color_eyre::Result { - Ok(Self(Database::import( - &format!( - "height_to_{}_txindex", - format!("{position:?}").to_lowercase() - ), - Self::version(), - )?)) - } - - pub fn insert(&mut self, height: Height, txindex: Txindex) { - self.0.insert(height.into(), txindex.into(), height) - } - - pub fn get(&self, height: Height) -> fjall::Result> { - self.0 - .get(Slice::from(height)) - .map(|opt| opt.map(|slice| slice.into())) - } - - pub fn remove(&mut self, height: Height) { - self.0.remove(Slice::from(height)) - } -} - -impl DatabaseTrait for HeightToTxindex { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/mod.rs b/src/crates/bitbase/src/database/mod.rs deleted file mode 100644 index 94b3469c2..000000000 --- a/src/crates/bitbase/src/database/mod.rs +++ /dev/null @@ -1,256 +0,0 @@ -use std::{collections::BTreeSet, ops::Sub, thread}; - -use biter::bitcoin::{hashes::Hash, BlockHash, Txid}; -use color_eyre::eyre::ContextCompat; -use fjall::Slice; - -mod addressbytes_prefix_to_addressindex; -mod addressindex_to_addressbytes; -mod addressindex_to_addresstype; -mod addressindex_to_txoutindexes; -mod blockhash_prefix_to_height; -mod height_to_blockhash; -mod height_to_txindex; -mod txid_prefix_to_txindex; -mod txindex_to_height; -mod txindex_to_txid; -mod txoutindex_to_addressindex; -mod txoutindex_to_amount; - -pub use addressbytes_prefix_to_addressindex::*; -pub use addressindex_to_addressbytes::*; -pub use addressindex_to_addresstype::*; -pub use addressindex_to_txoutindexes::*; -pub use blockhash_prefix_to_height::*; -pub use height_to_blockhash::*; -pub use height_to_txindex::*; -pub use txid_prefix_to_txindex::*; -pub use txindex_to_height::*; -pub use txindex_to_txid::*; -pub use txoutindex_to_addressindex::*; -pub use txoutindex_to_amount::*; - -use crate::structs::{Addressindex, Exit, Height, Txindex, Txoutindex}; - -pub struct Database { - pub addressbytes_prefix_to_addressindex: AddressbytesPrefixToAddressindex, - pub addressindex_to_addressbytes: AddressindexToAddressbytes, - pub addressindex_to_addresstype: AddressindexToAddresstype, - pub addressindex_to_txoutindexes: AddressindexToTxoutindexes, - pub blockhash_prefix_to_height: BlockhashPrefixToHeight, - pub height_to_blockhash: HeightToBlockhash, - pub height_to_first_txindex: HeightToTxindex, - pub height_to_last_txindex: HeightToTxindex, - pub txid_prefix_to_txindex: TxidPrefixToTxindex, - pub txindex_to_txid: TxindexToTxid, - pub txindex_to_height: TxindexToHeight, - pub txoutindex_to_addressindex: TxoutindexToAddressindex, - pub txoutindex_to_amount: TxoutindexToAmount, -} - -const UNSAFE_BLOCKS: usize = 100; - -impl Database { - pub fn import() -> color_eyre::Result { - thread::scope(|scope| { - let addressbytes_prefix_to_addressindex_handle = - scope.spawn(AddressbytesPrefixToAddressindex::import); - let addressindex_to_addressbytes_handle = - scope.spawn(AddressindexToAddressbytes::import); - let addressindex_to_addresstype_handle = scope.spawn(AddressindexToAddresstype::import); - let addressindex_to_txoutindexes_handle = - scope.spawn(AddressindexToTxoutindexes::import); - let blockhash_prefix_to_height_handle = scope.spawn(BlockhashPrefixToHeight::import); - let height_to_blockhash_handle = scope.spawn(HeightToBlockhash::import); - let height_to_first_txindex_handle = - scope.spawn(|| HeightToTxindex::import(HeightToTxindexPosition::First)); - let height_to_last_txindex_handle = - scope.spawn(|| HeightToTxindex::import(HeightToTxindexPosition::Last)); - let txid_prefix_to_txindex_handle = scope.spawn(TxidPrefixToTxindex::import); - let txindex_to_height_handle = scope.spawn(TxindexToHeight::import); - let txindex_to_txid_handle = scope.spawn(TxindexToTxid::import); - let txoutindex_to_addressindex_handle = scope.spawn(TxoutindexToAddressindex::import); - let txoutindex_to_amount_handle = scope.spawn(TxoutindexToAmount::import); - - Ok(Self { - addressbytes_prefix_to_addressindex: addressbytes_prefix_to_addressindex_handle - .join() - .unwrap()?, - addressindex_to_addressbytes: addressindex_to_addressbytes_handle - .join() - .unwrap()?, - addressindex_to_addresstype: addressindex_to_addresstype_handle.join().unwrap()?, - addressindex_to_txoutindexes: addressindex_to_txoutindexes_handle - .join() - .unwrap()?, - blockhash_prefix_to_height: blockhash_prefix_to_height_handle.join().unwrap()?, - height_to_blockhash: height_to_blockhash_handle.join().unwrap()?, - height_to_first_txindex: height_to_first_txindex_handle.join().unwrap()?, - height_to_last_txindex: height_to_last_txindex_handle.join().unwrap()?, - txid_prefix_to_txindex: txid_prefix_to_txindex_handle.join().unwrap()?, - txindex_to_height: txindex_to_height_handle.join().unwrap()?, - txindex_to_txid: txindex_to_txid_handle.join().unwrap()?, - txoutindex_to_addressindex: txoutindex_to_addressindex_handle.join().unwrap()?, - txoutindex_to_amount: txoutindex_to_amount_handle.join().unwrap()?, - }) - }) - } - - pub fn export(&mut self, height: Height) -> color_eyre::Result<()> { - thread::scope(|scope| { - scope.spawn(|| { - self.addressbytes_prefix_to_addressindex - .export(height) - .unwrap() - }); - scope.spawn(|| self.addressindex_to_addressbytes.export(height).unwrap()); - scope.spawn(|| self.addressindex_to_addresstype.export(height).unwrap()); - scope.spawn(|| self.addressindex_to_txoutindexes.export(height).unwrap()); - scope.spawn(|| self.blockhash_prefix_to_height.export(height).unwrap()); - scope.spawn(|| self.height_to_blockhash.export(height).unwrap()); - scope.spawn(|| self.height_to_first_txindex.export(height).unwrap()); - scope.spawn(|| self.height_to_last_txindex.export(height).unwrap()); - scope.spawn(|| self.txid_prefix_to_txindex.export(height).unwrap()); - scope.spawn(|| self.txindex_to_height.export(height).unwrap()); - scope.spawn(|| self.txindex_to_txid.export(height).unwrap()); - scope.spawn(|| self.txoutindex_to_addressindex.export(height).unwrap()); - scope.spawn(|| self.txoutindex_to_amount.export(height).unwrap()); - }); - Ok(()) - } - - pub fn start_height(&self) -> Height { - self.min_height() - .map(|h| h.sub(UNSAFE_BLOCKS)) - .unwrap_or_default() - } - - fn min_height(&self) -> Option { - [ - self.addressbytes_prefix_to_addressindex.height(), - self.addressindex_to_addressbytes.height(), - self.addressindex_to_addresstype.height(), - self.addressindex_to_txoutindexes.height(), - self.blockhash_prefix_to_height.height(), - self.height_to_blockhash.height(), - self.height_to_first_txindex.height(), - self.height_to_last_txindex.height(), - self.txid_prefix_to_txindex.height(), - self.txindex_to_height.height(), - self.txindex_to_txid.height(), - self.txoutindex_to_addressindex.height(), - self.txoutindex_to_amount.height(), - ] - .into_iter() - .map(ToOwned::to_owned) - .min() - .flatten() - } - - pub fn has_different_blockhash( - &self, - height: Height, - blockhash: &BlockHash, - ) -> fjall::Result { - Ok(self - .height_to_blockhash - .get(height)? - .is_some_and(|saved_blockhash| blockhash != &saved_blockhash)) - } - - pub fn rollback_from(&mut self, height: Height, exit: &Exit) -> color_eyre::Result<()> { - exit.block(); - - self.export(height)?; - - let mut txindex = None; - - self.height_to_blockhash - .range(Slice::from(height)..) - .try_for_each(|slice| -> color_eyre::Result<()> { - let (slice_height, slice_blockhash) = slice?; - let height = Height::from(slice_height); - let blockhash = BlockHash::from_slice(&slice_blockhash)?; - - self.height_to_blockhash.remove(height); - self.blockhash_prefix_to_height.remove(&blockhash); - if txindex.is_none() { - txindex.replace( - self.height_to_first_txindex - .get(height)? - .context("for height to have first txindex")?, - ); - } - self.height_to_first_txindex.remove(height); - self.height_to_last_txindex.remove(height); - - Ok(()) - })?; - - let txindex = txindex.context("txindex to not be none by now")?; - - self.txindex_to_txid - .range(Slice::from(txindex)..) - .try_for_each(|slice| -> color_eyre::Result<()> { - let (slice_txindex, slice_txid) = slice?; - let txindex = Txindex::from(slice_txindex); - let txid = Txid::from_slice(&slice_txid)?; - - self.txindex_to_txid.remove(txindex); - self.txindex_to_height.remove(txindex); - self.txid_prefix_to_txindex.remove(&txid); - - Ok(()) - })?; - - let txoutindex = Txoutindex::from(txindex); - - let mut addressindexes = BTreeSet::new(); - - self.txoutindex_to_amount - .range(Slice::from(txoutindex)..) - .try_for_each(|slice| -> color_eyre::Result<()> { - let (slice_txoutindex, _) = slice?; - let txoutindex = Txoutindex::from(slice_txoutindex); - - self.txoutindex_to_amount.remove(txoutindex); - - if let Some(addressindex_slice) = - self.txoutindex_to_addressindex.get(txoutindex.into())? - { - self.txoutindex_to_addressindex.remove(txoutindex); - - let addressindex = Addressindex::from(addressindex_slice); - addressindexes.insert(addressindex); - self.addressindex_to_txoutindexes - .remove(addressindex, txoutindex); - } - - Ok(()) - })?; - - self.export(height)?; - - addressindexes - .into_iter() - .filter(|addressindex| self.addressindex_to_txoutindexes.is_empty(*addressindex)) - .try_for_each(|addressindex| -> color_eyre::Result<()> { - let addressbytes = self - .addressindex_to_addressbytes - .get(addressindex)? - .context("addressindex_to_address to have value")?; - self.addressbytes_prefix_to_addressindex - .remove(&addressbytes); - self.addressindex_to_addressbytes.remove(addressindex); - self.addressindex_to_addresstype.remove(addressindex); - - Ok(()) - })?; - - self.export(height)?; - - exit.unblock(); - - Ok(()) - } -} diff --git a/src/crates/bitbase/src/database/txid_prefix_to_txindex.rs b/src/crates/bitbase/src/database/txid_prefix_to_txindex.rs deleted file mode 100644 index dfc1b6fab..000000000 --- a/src/crates/bitbase/src/database/txid_prefix_to_txindex.rs +++ /dev/null @@ -1,46 +0,0 @@ -use biter::bitcoin::Txid; -use color_eyre::eyre::eyre; -use derive_deref::{Deref, DerefMut}; -use fjall::Slice; - -use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version}; - -#[derive(Deref, DerefMut)] -pub struct TxidPrefixToTxindex(Database); - -impl TxidPrefixToTxindex { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "txid_prefix_to_txindex", - Self::version(), - )?)) - } - - pub fn insert( - &mut self, - txid: &Txid, - txindex: Txindex, - height: Height, - ) -> color_eyre::Result<()> { - if let Some(_txindex) = - self.fetch_update(Self::txid_to_key(txid), txindex.into(), height)? - { - return Err(eyre!("TxidPrefixToTxindex: key collision")); - } - Ok(()) - } - - pub fn remove(&mut self, txid: &Txid) { - self.0.remove(Self::txid_to_key(txid)) - } - - fn txid_to_key(txid: &Txid) -> Slice { - txid[0..8].into() - } -} - -impl DatabaseTrait for TxidPrefixToTxindex { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/txindex_to_height.rs b/src/crates/bitbase/src/database/txindex_to_height.rs deleted file mode 100644 index 32bb956a8..000000000 --- a/src/crates/bitbase/src/database/txindex_to_height.rs +++ /dev/null @@ -1,29 +0,0 @@ -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version}; - -#[derive(Deref, DerefMut)] -pub struct TxindexToHeight(Database); - -impl TxindexToHeight { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "txindex_to_height", - Self::version(), - )?)) - } - - pub fn insert(&mut self, txindex: Txindex, height: Height) { - self.0.insert(txindex.into(), height.into(), height) - } - - pub fn remove(&mut self, txindex: Txindex) { - self.0.remove(txindex.into()) - } -} - -impl DatabaseTrait for TxindexToHeight { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/txindex_to_txid.rs b/src/crates/bitbase/src/database/txindex_to_txid.rs deleted file mode 100644 index 7369fd6bf..000000000 --- a/src/crates/bitbase/src/database/txindex_to_txid.rs +++ /dev/null @@ -1,27 +0,0 @@ -use biter::bitcoin::Txid; -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version}; - -#[derive(Deref, DerefMut)] -pub struct TxindexToTxid(Database); - -impl TxindexToTxid { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import("txindex_to_txid", Self::version())?)) - } - - pub fn insert(&mut self, txindex: Txindex, txid: &Txid, height: Height) { - self.0.insert(txindex.into(), txid[..].into(), height) - } - - pub fn remove(&mut self, txindex: Txindex) { - self.0.remove(txindex.into()) - } -} - -impl DatabaseTrait for TxindexToTxid { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/txoutindex_to_addressindex.rs b/src/crates/bitbase/src/database/txoutindex_to_addressindex.rs deleted file mode 100644 index d71631db7..000000000 --- a/src/crates/bitbase/src/database/txoutindex_to_addressindex.rs +++ /dev/null @@ -1,30 +0,0 @@ -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Addressindex, Database, DatabaseTrait, Height, Txoutindex, Version}; - -#[derive(Deref, DerefMut)] -pub struct TxoutindexToAddressindex(Database); - -impl TxoutindexToAddressindex { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "txoutindex_to_addressindex", - Self::version(), - )?)) - } - - pub fn insert(&mut self, txoutindex: Txoutindex, addressindex: Addressindex, height: Height) { - self.0 - .insert(txoutindex.into(), addressindex.into(), height) - } - - pub fn remove(&mut self, txoutindex: Txoutindex) { - self.0.remove(txoutindex.into()) - } -} - -impl DatabaseTrait for TxoutindexToAddressindex { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/database/txoutindex_to_amount.rs b/src/crates/bitbase/src/database/txoutindex_to_amount.rs deleted file mode 100644 index 6a014d342..000000000 --- a/src/crates/bitbase/src/database/txoutindex_to_amount.rs +++ /dev/null @@ -1,29 +0,0 @@ -use derive_deref::{Deref, DerefMut}; - -use crate::structs::{Amount, Database, DatabaseTrait, Height, Txoutindex, Version}; - -#[derive(Deref, DerefMut)] -pub struct TxoutindexToAmount(Database); - -impl TxoutindexToAmount { - pub fn import() -> color_eyre::Result { - Ok(Self(Database::import( - "txoutindex_to_amount", - Self::version(), - )?)) - } - - pub fn insert(&mut self, txoutindex: Txoutindex, amount: Amount, height: Height) { - self.0.insert(txoutindex.into(), amount.into(), height) - } - - pub fn remove(&mut self, txoutindex: Txoutindex) { - self.0.remove(txoutindex.into()) - } -} - -impl DatabaseTrait for TxoutindexToAmount { - fn version() -> Version { - Version::from(1) - } -} diff --git a/src/crates/bitbase/src/main.rs b/src/crates/bitbase/src/main.rs index 5e42ee33d..8ad47a7e7 100644 --- a/src/crates/bitbase/src/main.rs +++ b/src/crates/bitbase/src/main.rs @@ -1,13 +1,18 @@ -use std::path::Path; +use std::{path::Path, str::FromStr}; -use biter::bitcoincore_rpc::{Auth, Client}; +use biter::{ + bitcoin::{hashes::Hash, Txid}, + bitcoincore_rpc::{Auth, Client}, +}; -mod database; mod structs; -use database::Database; -use fjall::Slice; -use structs::{Addressbytes, Addressindex, Addresstype, Exit, Height, Txindex, Txoutindex}; +use color_eyre::eyre::{eyre, ContextCompat}; +use fjall::{PersistMode, Slice, TransactionalKeyspace, WriteTransaction}; +use structs::{ + Addressbytes, Addressindex, Addresstxoutindex, Addresstype, Amount, Exit, Height, Partitions, + Prefix, SliceExtended, Txindex, Txoutindex, +}; // https://github.com/fjall-rs/fjall/discussions/72 // https://github.com/romanz/electrs/blob/master/doc/schema.md @@ -24,124 +29,241 @@ fn main() -> color_eyre::Result<()> { let exit = Exit::new(); - let mut db = Database::import()?; + let keyspace = fjall::Config::new("./database").open_transactional()?; - let mut height = db.start_height(); + let mut parts = Partitions::import(&keyspace, &exit)?; - let mut txindex = db - .height_to_last_txindex - .get(height)? + let wtx = keyspace.write_tx(); + + let mut height = parts.start_height(); + + let mut txindex = wtx + .get(parts.height_to_last_txindex.data(), Slice::from(height))? + .map(Txindex::from) .map(Txindex::incremented) .unwrap_or(Txindex::default()); - let mut addressindex = db - .txoutindex_to_addressindex - .prefix(Slice::from(txindex)) - .last() - .map(|res| -> color_eyre::Result { - Ok(Addressindex::from(res?.1).incremented()) - }) - .unwrap_or(Ok(Addressindex::default()))?; + let mut addressindex = wtx + .get( + parts.height_to_last_addressindex.data(), + Slice::from(height), + )? + .map(Addressindex::from) + .map(Addressindex::incremented) + .unwrap_or(Addressindex::default()); - let export = |db: &mut Database, height: Height| -> color_eyre::Result<()> { + let export = |keyspace: &TransactionalKeyspace, + mut wtx: WriteTransaction, + parts: &Partitions, + height: Height| + -> color_eyre::Result<()> { + parts.udpate_meta(&mut wtx, height); exit.block(); println!("Exporting..."); - db.export(height)?; + wtx.commit()?; + keyspace.persist(PersistMode::SyncAll)?; println!("Export done"); exit.unblock(); Ok(()) }; + let mut wtx_opt = Some(wtx); + biter::new(data_dir, Some(height.into()), None, rpc) .iter() .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { + let mut wtx = wtx_opt.take().context("option should've wtx")?; + println!("Processing block {_height}..."); height = Height::from(_height); - if db.has_different_blockhash(height, &blockhash)? { - db.rollback_from(height, &exit)?; + let has_different_blockhash = wtx + .get(parts.height_to_blockhash.data(), Slice::from(height))? + .is_some_and(|saved_blockhash_slice| blockhash[..] != saved_blockhash_slice[..]); + + if has_different_blockhash { + parts.rollback_from(&mut wtx, height, &exit)?; } - db.blockhash_prefix_to_height.insert(&blockhash, height)?; - db.height_to_blockhash.insert(height, &blockhash); + if parts.blockhash_prefix_to_height.needs(height) { + if let Some(prev) = wtx.fetch_update( + parts.blockhash_prefix_to_height.data(), + blockhash.prefix(), + |_| Some(Slice::from(height)), + )? { + dbg!(prev); + return Err(eyre!("Expect none")); + } + } + + if parts.height_to_blockhash.needs(height) { + wtx.insert( + parts.height_to_blockhash.data(), + Slice::from(height), + blockhash, + ); + } + + if parts.height_to_first_addressindex.needs(height) { + wtx.insert( + parts.height_to_first_addressindex.data(), + Slice::from(addressindex), + blockhash, + ); + } let txlen = block.txdata.len(); - let last_txindex = txlen - 1; + let last_txi = txlen - 1; block.txdata.into_iter().enumerate().try_for_each( |(txi, tx)| -> color_eyre::Result<()> { - if txi == 0 { - db.height_to_first_txindex.insert(height, txindex); + if txi == 0 && parts.height_to_first_txindex.needs(height) { + wtx.insert( + parts.height_to_first_txindex.data(), + Slice::from(height), + Slice::from(txindex), + ); } - if txi == last_txindex { - db.height_to_last_txindex.insert(height, txindex); + if txi == last_txi && parts.height_to_last_txindex.needs(height) { + wtx.insert( + parts.height_to_last_txindex.data(), + Slice::from(height), + Slice::from(txindex), + ); } - if !db.txindex_to_txid.is_safe(height) - || !db.txid_prefix_to_txindex.is_safe(height) + if parts.txindex_to_txid.needs(height) + || parts.txid_prefix_to_txindex.needs(height) { let txid = tx.compute_txid(); - db.txindex_to_txid.insert(txindex, &txid, height); - db.txid_prefix_to_txindex.insert(&txid, txindex, height)?; + + if parts.txindex_to_txid.needs(height) { + wtx.insert(parts.txindex_to_txid.data(), Slice::from(txindex), txid); + } + + if parts.txid_prefix_to_txindex.needs(height) { + if let Some(prev) = wtx.fetch_update( + parts.txid_prefix_to_txindex.data(), + txid.prefix(), + |_| Some(Slice::from(txindex)), + )? { + let prev_txid = Txid::from_slice(&wtx + .get(parts.txindex_to_txid.data(), &prev)?.expect("To have txid for txindex"))?; + + let only_known_dup_txids = [Txid::from_str("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599")?, Txid::from_str("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468")?]; + + // TODO + // link txindex to txid + // txid_prefix should point to the first vout so no override + // do not add vout as they're invalid + + if !only_known_dup_txids.contains(&prev_txid) { + let prev_height = Height::from(wtx.get(parts.txindex_to_height.data(), &prev)?.expect("To have height")); + let prev_txindex = Txindex::from(prev); + dbg!( + height, + txid, + txindex, + prev_height, + prev_txid, + prev_txindex, + ); + return Err(eyre!("Expect none")); + } + } + } } - db.txindex_to_height.insert(txindex, height); + if parts.txindex_to_height.needs(height) { + wtx.insert( + parts.txindex_to_height.data(), + Slice::from(txindex), + Slice::from(height), + ); + } txindex.increment(); - tx.output.into_iter().enumerate().try_for_each( + tx.output.iter().enumerate().try_for_each( |(vout, txout)| -> color_eyre::Result<()> { let vout = vout as u16; let txoutindex = Txoutindex::from((txindex, vout)); - let amount = txout.value.into(); - db.txoutindex_to_amount.insert(txoutindex, amount, height); + let amount = Amount::from(txout.value); + + if parts.txoutindex_to_amount.needs(height) { + wtx.insert( + parts.txoutindex_to_amount.data(), + Slice::from(txoutindex), + Slice::from(amount), + ); + } let script = &txout.script_pubkey; + let addresstype = Addresstype::from(script); let addressbytes = - Addressbytes::try_from(script).inspect_err(|_| { - dbg!(&txout, height, txi); - })?; - let addresstype = Addresstype::try_from(script)?; + Addressbytes::try_from((script, addresstype, addressindex)) + .inspect_err(|_| { + dbg!(&txout, height, txi, &tx.compute_txid()); + })?; let mut addressindex_local = addressindex; - if let Some(addressindex_slice) = db - .addressbytes_prefix_to_addressindex - .get((&addressbytes).into())? - { + if let Some(addressindex_slice) = wtx.get( + parts.addressbytes_prefix_to_addressindex.data(), + Slice::from(&addressbytes), + )? { addressindex_local = addressindex_slice.into() } else { - db.addressbytes_prefix_to_addressindex - .insert(&addressbytes, addressindex_local, height) - .inspect_err(|_| { - dbg!(addresstype); - })?; - db.addressindex_to_addressbytes.insert( - addressindex_local, - &addressbytes, - height, - ); - db.addressindex_to_addresstype.insert( - addressindex_local, - addresstype, - height, - ); + if parts.addressbytes_prefix_to_addressindex.needs(height) { + if let Some(prev) = wtx.fetch_update( + parts.addressbytes_prefix_to_addressindex.data(), + Slice::from(&addressbytes), + |_| Some(Slice::from(addressindex_local)), + )? { + dbg!(prev); + return Err(eyre!("Expect none")); + } + } + + if parts.addressindex_to_addressbytes.needs(height) { + wtx.insert( + parts.addressindex_to_addressbytes.data(), + Slice::from(addressindex_local), + Slice::from(&addressbytes), + ); + } + + if parts.addressindex_to_addresstype.needs(height) { + wtx.insert( + parts.addressindex_to_addresstype.data(), + Slice::from(addressindex_local), + Slice::from(addresstype), + ); + } + addressindex.increment(); } - db.txoutindex_to_addressindex.insert( - txoutindex, - addressindex_local, - height, - ); + if parts.txoutindex_to_addressindex.needs(height) { + wtx.insert( + parts.txoutindex_to_addressindex.data(), + Slice::from(txoutindex), + Slice::from(addressindex_local), + ); + } - db.addressindex_to_txoutindexes.insert( - addressindex_local, - txoutindex, - height, - ); + if parts.addresstxoutindexes.needs(height) { + wtx.insert( + parts.addresstxoutindexes.data(), + Slice::from(Addresstxoutindex::from(( + addressindex_local, + txoutindex, + ))), + Slice::default(), + ); + } Ok(()) }, @@ -151,15 +273,29 @@ fn main() -> color_eyre::Result<()> { }, )?; + if parts.height_to_last_addressindex.needs(height) { + wtx.insert( + parts.height_to_last_addressindex.data(), + Slice::from(addressindex.decremented()), + blockhash, + ); + } + let should_snapshot = _height % MONTHLY_BLOCK_TARGET == 0 && !exit.active(); if should_snapshot { - export(&mut db, height)?; + export(&keyspace, wtx, &parts, height)?; + wtx_opt.replace(keyspace.write_tx()); + } else { + wtx_opt.replace(wtx); } Ok(()) })?; - export(&mut db, height)?; + let wtx = wtx_opt + .take() + .context("option should have WriteTransaction")?; + export(&keyspace, wtx, &parts, height)?; dbg!(i.elapsed()); diff --git a/src/crates/bitbase/src/structs/addressbytes.rs b/src/crates/bitbase/src/structs/addressbytes.rs index f3533abd9..e51952e27 100644 --- a/src/crates/bitbase/src/structs/addressbytes.rs +++ b/src/crates/bitbase/src/structs/addressbytes.rs @@ -1,42 +1,62 @@ use biter::bitcoin::ScriptBuf; use color_eyre::eyre::eyre; +use derive_deref::{Deref, DerefMut}; use fjall::Slice; -#[derive(Debug)] +use super::{Addressindex, Addresstype}; + +#[derive(Debug, Deref, DerefMut)] pub struct Addressbytes(Slice); -impl Addressbytes { - pub fn to_prefix_slice(&self) -> Slice { - self.0[..8].into() - } -} - -impl TryFrom<&ScriptBuf> for Addressbytes { +impl TryFrom<(&ScriptBuf, Addresstype, Addressindex)> for Addressbytes { type Error = color_eyre::Report; - fn try_from(script: &ScriptBuf) -> Result { - if script.is_p2pk() { - let bytes = script.as_bytes(); - let bytes = match bytes.len() { - 67 => &script.as_bytes()[1..66], - 35 => &script.as_bytes()[1..34], - _ => { + fn try_from(tuple: (&ScriptBuf, Addresstype, Addressindex)) -> Result { + let (script, addresstype, addressindex) = tuple; + + match addresstype { + Addresstype::P2PK => { + let bytes = script.as_bytes(); + let bytes = match bytes.len() { + 67 => &script.as_bytes()[1..66], + 35 => &script.as_bytes()[1..34], + _ => { + dbg!(bytes); + return Err(eyre!("Wrong len")); + } + }; + + if bytes[0] != 4 { dbg!(bytes); - return Err(eyre!("Wrong len")); + return Err(eyre!("Doesn't start with a 4")); } - }; - if bytes[0] != 4 { - dbg!(bytes); - return Err(eyre!("Doesn't start with a 4")); + Ok(Self(bytes.into())) + } + Addresstype::P2PKH => { + let bytes = &script.as_bytes()[3..23]; + Ok(Self(bytes.into())) + } + _ => { + if script.is_p2sh() { + Err(eyre!("p2sh address type")) + } else if script.is_p2wpkh() { + Err(eyre!("p2wpkh address type")) + } else if script.is_p2wsh() { + Err(eyre!("p2wsh address type")) + } else if script.is_p2tr() { + Err(eyre!("p2tr address type")) + } else if script.is_empty() { + Err(eyre!("empty address type")) + } else if script.is_op_return() { + Err(eyre!("op_return address type")) + } else if script.is_multisig() { + Err(eyre!("multisig address type")) + } else if script.is_push_only() { + Err(eyre!("push only address type")) + } else { + Ok(Self(addressindex.into())) + } } - - Ok(Self(bytes.into())) - } else if script.is_p2pkh() { - let bytes = &script.as_bytes()[3..23]; - - Ok(Self(bytes.into())) - } else { - Err(eyre!("Unsupported address type")) } } } diff --git a/src/crates/bitbase/src/structs/addressindex.rs b/src/crates/bitbase/src/structs/addressindex.rs index 0d53e05d2..87fca33b2 100644 --- a/src/crates/bitbase/src/structs/addressindex.rs +++ b/src/crates/bitbase/src/structs/addressindex.rs @@ -9,6 +9,10 @@ pub struct Addressindex(u32); impl Addressindex { pub const BYTES: usize = size_of::(); + pub fn decremented(self) -> Self { + Self(*self - 1) + } + pub fn increment(&mut self) { self.0 += 1; } diff --git a/src/crates/bitbase/src/structs/addresstype.rs b/src/crates/bitbase/src/structs/addresstype.rs index 716c844d1..ff13060ac 100644 --- a/src/crates/bitbase/src/structs/addresstype.rs +++ b/src/crates/bitbase/src/structs/addresstype.rs @@ -4,21 +4,45 @@ use fjall::Slice; use super::SliceExtended; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Addresstype { P2PK, P2PKH, + P2SH, + P2WPKH, + P2WSH, + P2TR, + Multisig = 251, + PushOnly = 252, + OpReturn = 253, + Empty = 254, + Unknown = 255, } -impl TryFrom<&ScriptBuf> for Addresstype { - type Error = color_eyre::Report; - fn try_from(value: &ScriptBuf) -> Result { - if value.is_p2pk() { - Ok(Self::P2PK) - } else if value.is_p2pkh() { - Ok(Self::P2PKH) +impl From<&ScriptBuf> for Addresstype { + fn from(script: &ScriptBuf) -> Self { + if script.is_p2pk() { + Self::P2PK + } else if script.is_p2pkh() { + Self::P2PKH + } else if script.is_p2sh() { + Self::P2SH + } else if script.is_p2wpkh() { + Self::P2WPKH + } else if script.is_p2wsh() { + Self::P2WSH + } else if script.is_p2tr() { + Self::P2TR + } else if script.is_empty() { + Self::Empty + } else if script.is_op_return() { + Self::OpReturn + } else if script.is_push_only() { + Self::PushOnly + } else if script.is_multisig() { + Self::Multisig } else { - Err(eyre!("Not compatible script")) + Self::Unknown } } } diff --git a/src/crates/bitbase/src/structs/fjall.rs b/src/crates/bitbase/src/structs/fjall.rs deleted file mode 100644 index cfae90a45..000000000 --- a/src/crates/bitbase/src/structs/fjall.rs +++ /dev/null @@ -1,143 +0,0 @@ -use std::{mem, ops::RangeBounds}; - -pub use fjall::*; - -use crate::structs::{Height, Version}; - -pub struct Database { - keyspace: Keyspace, - data: PartitionHandle, - meta: PartitionHandle, - batch: Batch, - height: Option, -} - -const VERSION: &str = "version"; -const HEIGHT: &str = "height"; - -impl Database { - pub fn import(name: &str, version: Version) -> Result { - let keyspace = fjall::Config::new(format!("./database/{name}")).open()?; - - let data = Self::open_data(&keyspace)?; - let meta = Self::open_meta(&keyspace)?; - - let batch = keyspace.batch(); - - let mut this = Self { - height: meta.get(HEIGHT)?.map(Height::from), - keyspace, - data, - meta, - batch, - }; - - if let Some(slice) = this.meta.get(VERSION)? { - if version != Version::from(slice) { - this = this.reset()?; - } - } - - this.batch - .insert(&this.meta, VERSION, version.to_be_bytes()); - - Ok(this) - } - - fn open_data(keyspace: &Keyspace) -> Result { - keyspace.open_partition("data", Self::create_options()) - } - - fn open_meta(keyspace: &Keyspace) -> Result { - keyspace.open_partition("meta", Self::create_options()) - } - - fn create_options() -> PartitionCreateOptions { - PartitionCreateOptions::default().manual_journal_persist(true) - } - - pub fn get(&self, key: Slice) -> Result> { - self.data.get(key) - } - - pub fn range<'a, K: AsRef<[u8]> + 'a, R: RangeBounds + 'a>( - &'a self, - range: R, - ) -> impl DoubleEndedIterator> + 'static { - self.data.range(range) - } - - pub fn prefix<'a, K: AsRef<[u8]> + 'a>( - &'a self, - prefix: K, - ) -> impl DoubleEndedIterator> + 'static { - self.data.prefix(prefix) - } - - pub fn insert(&mut self, key: Slice, value: Slice, height: Height) { - if self.is_safe(height) { - return; - } - self.batch.insert(&self.data, key, value); - } - - pub fn fetch_update( - &mut self, - key: Slice, - value: Slice, - height: Height, - ) -> Result> { - if self.is_safe(height) { - return Ok(None); - } - let prev = self.get(key.clone()); - self.batch.insert(&self.data, key, value); - prev - } - - pub fn remove(&mut self, key: Slice) { - self.batch.remove(&self.data, key); - } - - pub fn is_safe(&self, height: Height) -> bool { - self.height.is_some_and(|self_height| self_height >= height) - } - - fn persist(&self) -> Result<()> { - self.keyspace.persist(PersistMode::SyncAll) - } - - pub fn export(&mut self, height: Height) -> Result<()> { - let mut batch = self.keyspace.batch(); - mem::swap(&mut batch, &mut self.batch); - - batch.insert(&self.meta, HEIGHT, height.to_be_bytes()); - - batch.commit()?; - - self.persist() - } - - fn reset(mut self) -> Result { - self.keyspace.delete_partition(self.data)?; - self.keyspace.delete_partition(self.meta)?; - - self.keyspace.persist(PersistMode::SyncAll)?; - - self.data = Self::open_data(&self.keyspace)?; - self.meta = Self::open_meta(&self.keyspace)?; - - Ok(self) - } - - pub fn height(&self) -> &Option { - &self.height - } -} - -pub trait DatabaseTrait -where - Self: Sized, -{ - fn version() -> Version; -} diff --git a/src/crates/bitbase/src/structs/mod.rs b/src/crates/bitbase/src/structs/mod.rs index ad6ef9954..6b557004e 100644 --- a/src/crates/bitbase/src/structs/mod.rs +++ b/src/crates/bitbase/src/structs/mod.rs @@ -4,8 +4,10 @@ mod addresstxoutindex; mod addresstype; mod amount; mod exit; -mod fjall; mod height; +mod partition; +mod partitions; +mod prefix; mod slice; mod txindex; mod txoutindex; @@ -17,8 +19,10 @@ pub use addresstxoutindex::*; pub use addresstype::*; pub use amount::*; pub use exit::*; -pub use fjall::*; pub use height::*; +pub use partition::*; +pub use partitions::*; +pub use prefix::*; pub use slice::*; pub use txindex::*; pub use txoutindex::*; diff --git a/src/crates/bitbase/src/structs/partition.rs b/src/crates/bitbase/src/structs/partition.rs new file mode 100644 index 000000000..bd913f22c --- /dev/null +++ b/src/crates/bitbase/src/structs/partition.rs @@ -0,0 +1,104 @@ +pub use fjall::{ + PartitionCreateOptions, PersistMode, Result, TransactionalKeyspace, + TransactionalPartitionHandle, +}; + +use crate::structs::{Height, Version}; + +use super::Exit; + +pub struct Partition { + version: Version, + data: TransactionalPartitionHandle, + meta: TransactionalPartitionHandle, + height: Option, +} + +impl Partition { + pub const VERSION: &str = "version"; + pub const HEIGHT: &str = "height"; + + pub fn import( + keyspace: &TransactionalKeyspace, + name: &str, + version: Version, + exit: &Exit, + ) -> Result { + let data = Self::open_data(keyspace, name)?; + let meta = Self::open_meta(keyspace, name)?; + + let mut this = Self { + version, + height: meta.get(Self::HEIGHT)?.map(Height::from), + data, + meta, + }; + + if let Some(slice) = this.meta.get(Self::VERSION)? { + if version != Version::from(slice) { + this = this.reset(keyspace, name, exit)?; + } + } + + Ok(this) + } + + fn open_data( + keyspace: &TransactionalKeyspace, + name: &str, + ) -> Result { + keyspace.open_partition(&format!("{name}-data"), Self::create_options()) + } + + fn open_meta( + keyspace: &TransactionalKeyspace, + name: &str, + ) -> Result { + keyspace.open_partition(&format!("{name}-meta"), Self::create_options()) + } + + fn create_options() -> PartitionCreateOptions { + PartitionCreateOptions::default().manual_journal_persist(true) + } + + pub fn is_safe(&self, height: Height) -> bool { + self.height.is_some_and(|self_height| self_height >= height) + } + + pub fn needs(&self, height: Height) -> bool { + !self.is_safe(height) + } + + pub fn version(&self) -> Version { + self.version + } + + pub fn data(&self) -> &TransactionalPartitionHandle { + &self.data + } + + pub fn meta(&self) -> &TransactionalPartitionHandle { + &self.meta + } + + fn reset(mut self, keyspace: &TransactionalKeyspace, name: &str, exit: &Exit) -> Result { + exit.block(); + + keyspace.delete_partition(self.data)?; + keyspace.delete_partition(self.meta)?; + + keyspace.persist(PersistMode::SyncAll)?; + + self.data = Self::open_data(keyspace, name)?; + self.meta = Self::open_meta(keyspace, name)?; + self.height = None; + + exit.unblock(); + + Ok(self) + } + + pub fn height(&self) -> &Option { + &self.height + } +} diff --git a/src/crates/bitbase/src/structs/partitions.rs b/src/crates/bitbase/src/structs/partitions.rs new file mode 100644 index 000000000..ebacd93c6 --- /dev/null +++ b/src/crates/bitbase/src/structs/partitions.rs @@ -0,0 +1,278 @@ +use std::ops::Sub; + +use fjall::{Slice, TransactionalKeyspace, WriteTransaction}; + +use crate::structs::{Exit, Height, Partition, Version}; + +pub struct Partitions { + pub addressbytes_prefix_to_addressindex: Partition, + pub addressindex_to_addressbytes: Partition, + pub addressindex_to_addresstype: Partition, + pub addresstxoutindexes: Partition, + pub blockhash_prefix_to_height: Partition, + pub height_to_blockhash: Partition, + pub height_to_first_addressindex: Partition, + pub height_to_first_txindex: Partition, + pub height_to_last_addressindex: Partition, + pub height_to_last_txindex: Partition, + pub txid_prefix_to_txindex: Partition, + pub txindex_to_height: Partition, + pub txindex_to_txid: Partition, + pub txoutindex_to_addressindex: Partition, + pub txoutindex_to_amount: Partition, +} + +const UNSAFE_BLOCKS: usize = 100; + +impl Partitions { + pub fn import(keyspace: &TransactionalKeyspace, exit: &Exit) -> color_eyre::Result { + Ok(Self { + addressbytes_prefix_to_addressindex: Partition::import( + keyspace, + "addressbytes_prefix_to_addressindex", + Version::from(1), + exit, + )?, + addressindex_to_addressbytes: Partition::import( + keyspace, + "addressindex_to_addressbytes", + Version::from(1), + exit, + )?, + addressindex_to_addresstype: Partition::import( + keyspace, + "addressindex_to_addresstype", + Version::from(1), + exit, + )?, + addresstxoutindexes: Partition::import( + keyspace, + "addresstxoutindexes", + Version::from(1), + exit, + )?, + blockhash_prefix_to_height: Partition::import( + keyspace, + "blockhash_prefix_to_height", + Version::from(1), + exit, + )?, + height_to_blockhash: Partition::import( + keyspace, + "height_to_blockhash", + Version::from(1), + exit, + )?, + height_to_first_addressindex: Partition::import( + keyspace, + "height_to_first_addressindex", + Version::from(1), + exit, + )?, + height_to_first_txindex: Partition::import( + keyspace, + "height_to_first_txindex", + Version::from(1), + exit, + )?, + height_to_last_addressindex: Partition::import( + keyspace, + "height_to_last_addressindex", + Version::from(1), + exit, + )?, + height_to_last_txindex: Partition::import( + keyspace, + "height_to_last_txindex", + Version::from(1), + exit, + )?, + txid_prefix_to_txindex: Partition::import( + keyspace, + "txid_prefix_to_txindex", + Version::from(1), + exit, + )?, + txindex_to_height: Partition::import( + keyspace, + "txindex_to_height", + Version::from(1), + exit, + )?, + txindex_to_txid: Partition::import( + keyspace, + "txindex_to_txid", + Version::from(1), + exit, + )?, + txoutindex_to_addressindex: Partition::import( + keyspace, + "txoutindex_to_addressindex", + Version::from(1), + exit, + )?, + txoutindex_to_amount: Partition::import( + keyspace, + "txoutindex_to_amount", + Version::from(1), + exit, + )?, + }) + } + + pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) { + self.to_vec().into_iter().for_each(|part| { + let meta = part.meta(); + wtx.insert(meta, Partition::VERSION, Slice::from(part.version())); + wtx.insert(meta, Partition::HEIGHT, height.to_be_bytes()); + }); + } + + pub fn start_height(&self) -> Height { + self.min_height() + .map(|height| height.sub(UNSAFE_BLOCKS)) + .unwrap_or_default() + } + + fn min_height(&self) -> Option { + self.to_vec() + .into_iter() + .map(|part| part.height()) + .map(ToOwned::to_owned) + .min() + .flatten() + } + + pub fn rollback_from( + &mut self, + _wtx: &mut WriteTransaction, + _height: Height, + _exit: &Exit, + ) -> color_eyre::Result<()> { + panic!(); + // let mut txindex = None; + + // wtx.range(self.height_to_blockhash.data(), Slice::from(height)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (height_slice, slice_blockhash) = slice?; + // let blockhash = BlockHash::from_slice(&slice_blockhash)?; + + // wtx.remove(self.height_to_blockhash.data(), height_slice); + + // wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix()); + + // if txindex.is_none() { + // txindex.replace( + // wtx.get(self.height_to_first_txindex.data(), height_slice)? + // .context("for height to have first txindex")?, + // ); + // } + // wtx.remove(self.height_to_first_txindex.data(), height_slice); + // wtx.remove(self.height_to_last_txindex.data(), height_slice); + + // Ok(()) + // })?; + + // let txindex = txindex.context("txindex to not be none by now")?; + + // wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (slice_txindex, slice_txid) = slice?; + // let txindex = Txindex::from(slice_txindex); + // let txid = Txid::from_slice(&slice_txid)?; + + // wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex)); + // wtx.remove(self.txindex_to_height.data(), Slice::from(txindex)); + // wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix()); + + // Ok(()) + // })?; + + // let txoutindex = Txoutindex::from(txindex); + + // let mut addressindexes = BTreeSet::new(); + + // wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (txoutindex_slice, _) = slice?; + + // wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice); + + // if let Some(addressindex_slice) = + // wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)? + // { + // wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice); + + // let addressindex = Addressindex::from(addressindex_slice); + // addressindexes.insert(addressindex); + + // let txoutindex = Txoutindex::from(txoutindex_slice); + // let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex)); + + // wtx.remove( + // self.addressindex_to_txoutindexes.data(), + // Slice::from(addresstxoutindex), + // ); + // } + + // Ok(()) + // })?; + + // addressindexes + // .into_iter() + // .filter(|addressindex| { + // let is_empty = wtx + // .prefix( + // self.addressindex_to_txoutindexes.data(), + // Slice::from(*addressindex), + // ) + // .next() + // .is_none(); + // is_empty + // }) + // .try_for_each(|addressindex| -> color_eyre::Result<()> { + // let addressindex_slice = Slice::from(addressindex); + + // let addressbytes = Addressbytes::from( + // wtx.get( + // self.addressindex_to_addressbytes.data(), + // &addressindex_slice, + // )? + // .context("addressindex_to_address to have value")?, + // ); + // wtx.remove( + // self.addressbytes_prefix_to_addressindex.data(), + // addressbytes.prefix(), + // ); + // wtx.remove( + // self.addressindex_to_addressbytes.data(), + // &addressindex_slice, + // ); + // wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice); + + // Ok(()) + // })?; + + // Ok(()) + } + + fn to_vec(&self) -> Vec<&Partition> { + vec![ + &self.addressbytes_prefix_to_addressindex, + &self.addressindex_to_addressbytes, + &self.addressindex_to_addresstype, + &self.addresstxoutindexes, + &self.blockhash_prefix_to_height, + &self.height_to_blockhash, + &self.height_to_first_addressindex, + &self.height_to_first_txindex, + &self.height_to_last_addressindex, + &self.height_to_last_txindex, + &self.txid_prefix_to_txindex, + &self.txindex_to_height, + &self.txindex_to_txid, + &self.txoutindex_to_addressindex, + &self.txoutindex_to_amount, + ] + } +} diff --git a/src/crates/bitbase/src/structs/prefix.rs b/src/crates/bitbase/src/structs/prefix.rs new file mode 100644 index 000000000..951979107 --- /dev/null +++ b/src/crates/bitbase/src/structs/prefix.rs @@ -0,0 +1,25 @@ +use biter::bitcoin::{BlockHash, Txid}; + +use super::Addressbytes; + +pub trait Prefix { + fn prefix(&self) -> &[u8]; +} + +impl Prefix for Addressbytes { + fn prefix(&self) -> &[u8] { + &self[..8] + } +} + +impl Prefix for BlockHash { + fn prefix(&self) -> &[u8] { + &self[..8] + } +} + +impl Prefix for Txid { + fn prefix(&self) -> &[u8] { + &self[..8] + } +} diff --git a/src/crates/bitbase/src/structs/version.rs b/src/crates/bitbase/src/structs/version.rs index 0c4692388..80fdf235c 100644 --- a/src/crates/bitbase/src/structs/version.rs +++ b/src/crates/bitbase/src/structs/version.rs @@ -17,3 +17,9 @@ impl From for Version { Self(slice.read_u8()) } } + +impl From for Slice { + fn from(value: Version) -> Self { + value.to_be_bytes().into() + } +} diff --git a/src/crates/biter/src/lib.rs b/src/crates/biter/src/lib.rs index 55daa343e..d71d93027 100644 --- a/src/crates/biter/src/lib.rs +++ b/src/crates/biter/src/lib.rs @@ -77,7 +77,7 @@ enum BlockState { /// }); /// /// dbg!(i.elapsed()); -///} +/// } /// ``` /// pub fn new(