From 1380b42c1de374da00524b7c7e233aaf2fcdf832 Mon Sep 17 00:00:00 2001 From: nym21 Date: Fri, 3 Jan 2025 21:34:10 +0100 Subject: [PATCH] bitbase: snapshot --- Cargo.lock | 12 +- src/crates/barser/src/main.rs | 100 ----------- src/crates/barser/src/structs/txoutindex.rs | 53 ------ src/crates/bitbase/.gitignore | 1 + src/crates/{barser => bitbase}/Cargo.lock | 79 ++++++--- src/crates/{barser => bitbase}/Cargo.toml | 3 +- .../addressbytes_prefix_to_addressindex.rs | 41 +++++ .../database/addressindex_to_addressbytes.rs | 43 +++++ .../database/addressindex_to_addresstype.rs | 38 ++++ .../database/addressindex_to_txoutindexes.rs | 42 +++++ .../database}/blockhash_prefix_to_height.rs | 0 .../src/database}/height_to_blockhash.rs | 0 .../src/database}/height_to_txindex.rs | 0 .../databases => bitbase/src/database}/mod.rs | 119 +++++++++++-- .../src/database}/txid_prefix_to_txindex.rs | 0 .../bitbase/src/database/txindex_to_height.rs | 29 +++ .../src/database}/txindex_to_txid.rs | 3 +- .../database/txoutindex_to_addressindex.rs | 30 ++++ .../src/database}/txoutindex_to_amount.rs | 3 +- src/crates/bitbase/src/main.rs | 167 ++++++++++++++++++ .../bitbase/src/structs/addressbytes.rs | 53 ++++++ .../bitbase/src/structs/addressindex.rs | 47 +++++ .../bitbase/src/structs/addresstxoutindex.rs | 35 ++++ src/crates/bitbase/src/structs/addresstype.rs | 40 +++++ .../{barser => bitbase}/src/structs/amount.rs | 3 +- .../{barser => bitbase}/src/structs/exit.rs | 0 .../src/structs/fjall.rs} | 15 +- .../{barser => bitbase}/src/structs/height.rs | 7 +- .../{barser => bitbase}/src/structs/mod.rs | 12 +- .../{barser => bitbase}/src/structs/slice.rs | 17 +- .../src/structs/txindex.rs | 9 +- src/crates/bitbase/src/structs/txoutindex.rs | 46 +++++ .../src/structs/version.rs | 5 +- src/crates/biter/Cargo.lock | 24 +-- src/crates/biter/Cargo.toml | 4 +- src/crates/biter/src/blk_index_to_blk_path.rs | 46 +++++ .../biter/src/blk_index_to_blk_recap.rs | 6 +- src/crates/biter/src/lib.rs | 12 +- src/crates/biter/src/utils.rs | 39 +--- src/parser/actions/iter_blocks.rs | 2 +- 40 files changed, 906 insertions(+), 279 deletions(-) delete mode 100644 src/crates/barser/src/main.rs delete mode 100644 src/crates/barser/src/structs/txoutindex.rs create mode 100644 src/crates/bitbase/.gitignore rename src/crates/{barser => bitbase}/Cargo.lock (95%) rename src/crates/{barser => bitbase}/Cargo.toml (79%) create mode 100644 src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs create mode 100644 src/crates/bitbase/src/database/addressindex_to_addressbytes.rs create mode 100644 src/crates/bitbase/src/database/addressindex_to_addresstype.rs create mode 100644 src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs rename src/crates/{barser/src/databases => bitbase/src/database}/blockhash_prefix_to_height.rs (100%) rename src/crates/{barser/src/databases => bitbase/src/database}/height_to_blockhash.rs (100%) rename src/crates/{barser/src/databases => bitbase/src/database}/height_to_txindex.rs (100%) rename src/crates/{barser/src/databases => bitbase/src/database}/mod.rs (53%) rename src/crates/{barser/src/databases => bitbase/src/database}/txid_prefix_to_txindex.rs (100%) create mode 100644 src/crates/bitbase/src/database/txindex_to_height.rs rename src/crates/{barser/src/databases => bitbase/src/database}/txindex_to_txid.rs (91%) create mode 100644 src/crates/bitbase/src/database/txoutindex_to_addressindex.rs rename src/crates/{barser/src/databases => bitbase/src/database}/txoutindex_to_amount.rs (91%) create mode 100644 src/crates/bitbase/src/main.rs create mode 100644 src/crates/bitbase/src/structs/addressbytes.rs create mode 100644 src/crates/bitbase/src/structs/addressindex.rs create mode 100644 src/crates/bitbase/src/structs/addresstxoutindex.rs create mode 100644 src/crates/bitbase/src/structs/addresstype.rs rename src/crates/{barser => bitbase}/src/structs/amount.rs (97%) rename src/crates/{barser => bitbase}/src/structs/exit.rs (100%) rename src/crates/{barser/src/structs/database.rs => bitbase/src/structs/fjall.rs} (90%) rename src/crates/{barser => bitbase}/src/structs/height.rs (94%) rename src/crates/{barser => bitbase}/src/structs/mod.rs (51%) rename src/crates/{barser => bitbase}/src/structs/slice.rs (70%) rename src/crates/{barser => bitbase}/src/structs/txindex.rs (81%) create mode 100644 src/crates/bitbase/src/structs/txoutindex.rs rename src/crates/{barser => bitbase}/src/structs/version.rs (77%) create mode 100644 src/crates/biter/src/blk_index_to_blk_path.rs diff --git a/Cargo.lock b/Cargo.lock index 4122381e8..66ae12c36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2484,18 +2484,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -2504,9 +2504,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", diff --git a/src/crates/barser/src/main.rs b/src/crates/barser/src/main.rs deleted file mode 100644 index e3ce5399e..000000000 --- a/src/crates/barser/src/main.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::path::Path; - -use biter::bitcoincore_rpc::{Auth, Client}; - -mod databases; -mod structs; - -use databases::Databases; -use structs::{Exit, Height, Txindex, Txoutindex}; - -// https://github.com/fjall-rs/fjall/discussions/72 -// https://github.com/romanz/electrs/blob/master/doc/schema.md - -const DAILY_BLOCK_TARGET: usize = 144; -const MONTHLY_BLOCK_TARGET: usize = DAILY_BLOCK_TARGET * 30; - -fn main() -> color_eyre::Result<()> { - let i = std::time::Instant::now(); - - let data_dir = Path::new("../bitcoin"); - let cookie = Path::new(data_dir).join(".cookie"); - let rpc = Client::new("http://localhost:8332", Auth::CookieFile(cookie)).unwrap(); - - let exit = Exit::new(); - - let mut dbs = Databases::import()?; - - let mut height = dbs.start_height(&rpc)?; - - let mut txindex = dbs - .height_to_last_txindex - .get(height)? - .unwrap_or(Txindex::default()); - - let export = |dbs: &mut Databases, height: Height| -> color_eyre::Result<()> { - exit.block(); - println!("Exporting..."); - dbs.export(height)?; - println!("Export done"); - exit.unblock(); - Ok(()) - }; - - biter::new(data_dir, Some(height.into()), None, rpc) - .iter() - .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { - println!("Processing block {_height}..."); - - height = Height::from(_height); - - if dbs.has_different_blockhash(height, &blockhash)? { - dbs.erase_from(height)?; - } - - dbs.blockhash_prefix_to_height.insert(&blockhash, height)?; - dbs.height_to_blockhash.insert(height, &blockhash); - - let txlen = block.txdata.len(); - - block.txdata.into_iter().enumerate().try_for_each( - |(i, tx)| -> color_eyre::Result<()> { - if i == txlen - 1 { - dbs.height_to_last_txindex.insert(height, txindex); - } - - if !dbs.txindex_to_txid.is_safe(height) - || !dbs.txid_prefix_to_txindex.is_safe(height) - { - let txid = tx.compute_txid(); - dbs.txindex_to_txid.insert(txindex, &txid, height); - dbs.txid_prefix_to_txindex.insert(&txid, txindex, height)?; - } - - txindex.increment(); - - tx.output.into_iter().enumerate().for_each(|(vout, txout)| { - let vout = vout as u16; - let txoutindex = Txoutindex::from((txindex, vout)); - let amount = txout.value.into(); - dbs.txoutindex_to_amount.insert(txoutindex, amount, height); - }); - - Ok(()) - }, - )?; - - let should_snapshot = _height % MONTHLY_BLOCK_TARGET == 0 && !exit.active(); - if should_snapshot { - export(&mut dbs, height)?; - } - - Ok(()) - })?; - - export(&mut dbs, height)?; - - dbg!(i.elapsed()); - - Ok(()) -} diff --git a/src/crates/barser/src/structs/txoutindex.rs b/src/crates/barser/src/structs/txoutindex.rs deleted file mode 100644 index ab05801ad..000000000 --- a/src/crates/barser/src/structs/txoutindex.rs +++ /dev/null @@ -1,53 +0,0 @@ -use super::{SliceExtended, Txindex}; - -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)] -pub struct Txoutindex { - pub txindex: Txindex, - pub vout: u16, -} - -const SHIFT: u64 = 16; -const AND: u64 = (1 << SHIFT) - 1; - -impl From for Txoutindex { - fn from(value: Txindex) -> Self { - Self { - txindex: value, - vout: 0, - } - } -} - -impl From<(Txindex, u16)> for Txoutindex { - fn from(value: (Txindex, u16)) -> Self { - Self { - txindex: value.0, - vout: value.1, - } - } -} - -impl From for Txoutindex { - fn from(value: u64) -> Self { - Self { - txindex: (value >> SHIFT).into(), - vout: (value & AND) as u16, - } - } -} -impl From for u64 { - fn from(value: Txoutindex) -> Self { - (u64::from(value.txindex) << SHIFT) + value.vout as u64 - } -} - -impl From for fjall::Slice { - fn from(value: Txoutindex) -> Self { - u64::from(value).to_be_bytes().into() - } -} -impl From for Txoutindex { - fn from(value: fjall::Slice) -> Self { - fjall::Slice::read_u64(&value).into() - } -} diff --git a/src/crates/bitbase/.gitignore b/src/crates/bitbase/.gitignore new file mode 100644 index 000000000..e2647cd44 --- /dev/null +++ b/src/crates/bitbase/.gitignore @@ -0,0 +1 @@ +/database diff --git a/src/crates/barser/Cargo.lock b/src/crates/bitbase/Cargo.lock similarity index 95% rename from src/crates/barser/Cargo.lock rename to src/crates/bitbase/Cargo.lock index 7d20da2a9..1cb32bd0d 100644 --- a/src/crates/barser/Cargo.lock +++ b/src/crates/bitbase/Cargo.lock @@ -50,8 +50,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f" dependencies = [ - "bitcoin-internals", - "bitcoin_hashes", + "bitcoin-internals 0.3.0", + "bitcoin_hashes 0.14.0", ] [[package]] @@ -66,6 +66,19 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" +[[package]] +name = "bitbase" +version = "0.1.0" +dependencies = [ + "bitcoin_hashes 0.16.0", + "biter", + "color-eyre", + "ctrlc", + "derive_deref", + "fjall", + "rayon", +] + [[package]] name = "bitcoin" version = "0.32.5" @@ -74,11 +87,11 @@ checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026" dependencies = [ "base58ck", "bech32", - "bitcoin-internals", - "bitcoin-io", + "bitcoin-internals 0.3.0", + "bitcoin-io 0.1.3", "bitcoin-units", - "bitcoin_hashes", - "hex-conservative", + "bitcoin_hashes 0.14.0", + "hex-conservative 0.2.1", "hex_lit", "secp256k1", "serde", @@ -93,19 +106,34 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcoin-internals" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b854212e29b96c8f0fe04cab11d57586c8f3257de0d146c76cb3b42b3eb9118" + [[package]] name = "bitcoin-io" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" +[[package]] +name = "bitcoin-io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26792cd2bf245069a1c5acb06aa7ad7abe1de69b507c90b490bca81e0665d0ee" +dependencies = [ + "bitcoin-internals 0.4.0", +] + [[package]] name = "bitcoin-units" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2" dependencies = [ - "bitcoin-internals", + "bitcoin-internals 0.3.0", "serde", ] @@ -115,11 +143,21 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" dependencies = [ - "bitcoin-io", - "hex-conservative", + "bitcoin-io 0.1.3", + "hex-conservative 0.2.1", "serde", ] +[[package]] +name = "bitcoin_hashes" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e5d09f16329cd545d7e6008b2c6b2af3a90bc678cf41ac3d2f6755943301b16" +dependencies = [ + "bitcoin-io 0.2.0", + "hex-conservative 0.3.0", +] + [[package]] name = "bitcoincore-rpc" version = "0.19.0" @@ -441,6 +479,15 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "hex-conservative" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afe881d0527571892c4034822e59bb10c6c991cce6abe8199b6f5cf10766f55" +dependencies = [ + "arrayvec", +] + [[package]] name = "hex_lit" version = "0.1.1" @@ -732,18 +779,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "rust-playground" -version = "0.1.0" -dependencies = [ - "biter", - "color-eyre", - "ctrlc", - "derive_deref", - "fjall", - "rayon", -] - [[package]] name = "rustc-demangle" version = "0.1.24" @@ -787,7 +822,7 @@ version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ - "bitcoin_hashes", + "bitcoin_hashes 0.14.0", "rand", "secp256k1-sys", "serde", diff --git a/src/crates/barser/Cargo.toml b/src/crates/bitbase/Cargo.toml similarity index 79% rename from src/crates/barser/Cargo.toml rename to src/crates/bitbase/Cargo.toml index 9902da560..0febf661b 100644 --- a/src/crates/barser/Cargo.toml +++ b/src/crates/bitbase/Cargo.toml @@ -1,9 +1,10 @@ [package] -name = "barser" +name = "bitbase" version = "0.1.0" edition = "2021" [dependencies] +bitcoin_hashes = "0.16.0" biter = "0.2.2" color-eyre = "0.6.3" ctrlc = "3.4.5" diff --git a/src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs b/src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs new file mode 100644 index 000000000..a72cee62a --- /dev/null +++ b/src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs @@ -0,0 +1,41 @@ +use color_eyre::eyre::eyre; +use derive_deref::{Deref, DerefMut}; + +use crate::structs::{Addressbytes, Addressindex, Database, DatabaseTrait, Height, Version}; + +#[derive(Deref, DerefMut)] +pub struct AddressbytesPrefixToAddressindex(Database); + +impl AddressbytesPrefixToAddressindex { + pub fn import() -> color_eyre::Result { + Ok(Self(Database::import( + "address_prefix_to_addressindex", + Self::version(), + )?)) + } + + pub fn insert( + &mut self, + addressbytes: &Addressbytes, + addressindex: Addressindex, + height: Height, + ) -> color_eyre::Result<()> { + if let Some(_height) = + self.fetch_update(addressbytes.to_prefix_slice(), addressindex.into(), height)? + { + dbg!(addressbytes, addressindex); + return Err(eyre!("AddressPrefixToAddressindex: key collision")); + } + Ok(()) + } + + pub fn remove(&mut self, addressbytes: &Addressbytes) { + self.0.remove(addressbytes.to_prefix_slice()) + } +} + +impl DatabaseTrait for AddressbytesPrefixToAddressindex { + fn version() -> Version { + Version::from(1) + } +} diff --git a/src/crates/bitbase/src/database/addressindex_to_addressbytes.rs b/src/crates/bitbase/src/database/addressindex_to_addressbytes.rs new file mode 100644 index 000000000..228bc3965 --- /dev/null +++ b/src/crates/bitbase/src/database/addressindex_to_addressbytes.rs @@ -0,0 +1,43 @@ +use derive_deref::{Deref, DerefMut}; + +use crate::structs::{Addressbytes, Addressindex, Database, DatabaseTrait, Height, Version}; + +#[derive(Deref, DerefMut)] +pub struct AddressindexToAddressbytes(Database); + +impl AddressindexToAddressbytes { + pub fn import() -> color_eyre::Result { + Ok(Self(Database::import( + "addressindex_to_addressbytes", + Self::version(), + )?)) + } + + pub fn get(&self, addressindex: Addressindex) -> color_eyre::Result> { + if let Some(address) = self.0.get(addressindex.into())?.map(Addressbytes::try_from) { + Ok(Some(address?)) + } else { + Ok(None) + } + } + + pub fn insert( + &mut self, + addressindex: Addressindex, + addressbytes: &Addressbytes, + height: Height, + ) { + self.0 + .insert(addressindex.into(), addressbytes.into(), height) + } + + pub fn remove(&mut self, addressindex: Addressindex) { + self.0.remove(addressindex.into()) + } +} + +impl DatabaseTrait for AddressindexToAddressbytes { + fn version() -> Version { + Version::from(1) + } +} diff --git a/src/crates/bitbase/src/database/addressindex_to_addresstype.rs b/src/crates/bitbase/src/database/addressindex_to_addresstype.rs new file mode 100644 index 000000000..32fc0046a --- /dev/null +++ b/src/crates/bitbase/src/database/addressindex_to_addresstype.rs @@ -0,0 +1,38 @@ +use derive_deref::{Deref, DerefMut}; + +use crate::structs::{Addressindex, Addresstype, Database, DatabaseTrait, Height, Version}; + +#[derive(Deref, DerefMut)] +pub struct AddressindexToAddresstype(Database); + +impl AddressindexToAddresstype { + pub fn import() -> color_eyre::Result { + Ok(Self(Database::import( + "addressindex_to_addresstype", + Self::version(), + )?)) + } + + // pub fn get(&self, addressindex: Addressindex) -> color_eyre::Result> { + // if let Some(addresstype) = self.0.get(addressindex.into())?.map(Addresstype::try_from) { + // Ok(Some(addresstype?)) + // } else { + // Ok(None) + // } + // } + + pub fn insert(&mut self, addressindex: Addressindex, addresstype: Addresstype, height: Height) { + self.0 + .insert(addressindex.into(), addresstype.into(), height) + } + + pub fn remove(&mut self, addressindex: Addressindex) { + self.0.remove(addressindex.into()) + } +} + +impl DatabaseTrait for AddressindexToAddresstype { + fn version() -> Version { + Version::from(1) + } +} diff --git a/src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs b/src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs new file mode 100644 index 000000000..7f173c3a9 --- /dev/null +++ b/src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs @@ -0,0 +1,42 @@ +use derive_deref::{Deref, DerefMut}; +use fjall::Slice; + +use crate::structs::{ + Addressindex, Addresstxoutindex, Database, DatabaseTrait, Height, SliceExtended, Txoutindex, + Version, +}; + +#[derive(Deref, DerefMut)] +pub struct AddressindexToTxoutindexes(Database); + +impl AddressindexToTxoutindexes { + pub fn import() -> color_eyre::Result { + Ok(Self(Database::import( + "addressindex_to_txoutindexes", + Self::version(), + )?)) + } + + pub fn insert(&mut self, addressindex: Addressindex, txoutindex: Txoutindex, height: Height) { + self.0.insert( + Addresstxoutindex::from((addressindex, txoutindex)).into(), + Slice::default(), + height, + ) + } + + pub fn remove(&mut self, addressindex: Addressindex, txoutindex: Txoutindex) { + self.0 + .remove(Addresstxoutindex::from((addressindex, txoutindex)).into()); + } + + pub fn is_empty(&self, addressindex: Addressindex) -> bool { + self.prefix(Slice::from(addressindex)).next().is_none() + } +} + +impl DatabaseTrait for AddressindexToTxoutindexes { + fn version() -> Version { + Version::from(1) + } +} diff --git a/src/crates/barser/src/databases/blockhash_prefix_to_height.rs b/src/crates/bitbase/src/database/blockhash_prefix_to_height.rs similarity index 100% rename from src/crates/barser/src/databases/blockhash_prefix_to_height.rs rename to src/crates/bitbase/src/database/blockhash_prefix_to_height.rs diff --git a/src/crates/barser/src/databases/height_to_blockhash.rs b/src/crates/bitbase/src/database/height_to_blockhash.rs similarity index 100% rename from src/crates/barser/src/databases/height_to_blockhash.rs rename to src/crates/bitbase/src/database/height_to_blockhash.rs diff --git a/src/crates/barser/src/databases/height_to_txindex.rs b/src/crates/bitbase/src/database/height_to_txindex.rs similarity index 100% rename from src/crates/barser/src/databases/height_to_txindex.rs rename to src/crates/bitbase/src/database/height_to_txindex.rs diff --git a/src/crates/barser/src/databases/mod.rs b/src/crates/bitbase/src/database/mod.rs similarity index 53% rename from src/crates/barser/src/databases/mod.rs rename to src/crates/bitbase/src/database/mod.rs index fd9066db9..94b3469c2 100644 --- a/src/crates/barser/src/databases/mod.rs +++ b/src/crates/bitbase/src/database/mod.rs @@ -1,43 +1,65 @@ -use std::{ops::Sub, thread}; +use std::{collections::BTreeSet, ops::Sub, thread}; -use biter::{ - bitcoin::{hashes::Hash, BlockHash, Txid}, - bitcoincore_rpc::Client, -}; -pub use blockhash_prefix_to_height::*; +use biter::bitcoin::{hashes::Hash, BlockHash, Txid}; use color_eyre::eyre::ContextCompat; use fjall::Slice; +mod addressbytes_prefix_to_addressindex; +mod addressindex_to_addressbytes; +mod addressindex_to_addresstype; +mod addressindex_to_txoutindexes; mod blockhash_prefix_to_height; mod height_to_blockhash; mod height_to_txindex; mod txid_prefix_to_txindex; +mod txindex_to_height; mod txindex_to_txid; +mod txoutindex_to_addressindex; mod txoutindex_to_amount; +pub use addressbytes_prefix_to_addressindex::*; +pub use addressindex_to_addressbytes::*; +pub use addressindex_to_addresstype::*; +pub use addressindex_to_txoutindexes::*; +pub use blockhash_prefix_to_height::*; pub use height_to_blockhash::*; pub use height_to_txindex::*; pub use txid_prefix_to_txindex::*; +pub use txindex_to_height::*; pub use txindex_to_txid::*; +pub use txoutindex_to_addressindex::*; pub use txoutindex_to_amount::*; -use crate::structs::{Height, Txindex, Txoutindex}; +use crate::structs::{Addressindex, Exit, Height, Txindex, Txoutindex}; -pub struct Databases { +pub struct Database { + pub addressbytes_prefix_to_addressindex: AddressbytesPrefixToAddressindex, + pub addressindex_to_addressbytes: AddressindexToAddressbytes, + pub addressindex_to_addresstype: AddressindexToAddresstype, + pub addressindex_to_txoutindexes: AddressindexToTxoutindexes, pub blockhash_prefix_to_height: BlockhashPrefixToHeight, pub height_to_blockhash: HeightToBlockhash, pub height_to_first_txindex: HeightToTxindex, pub height_to_last_txindex: HeightToTxindex, pub txid_prefix_to_txindex: TxidPrefixToTxindex, pub txindex_to_txid: TxindexToTxid, + pub txindex_to_height: TxindexToHeight, + pub txoutindex_to_addressindex: TxoutindexToAddressindex, pub txoutindex_to_amount: TxoutindexToAmount, } const UNSAFE_BLOCKS: usize = 100; -impl Databases { +impl Database { pub fn import() -> color_eyre::Result { thread::scope(|scope| { + let addressbytes_prefix_to_addressindex_handle = + scope.spawn(AddressbytesPrefixToAddressindex::import); + let addressindex_to_addressbytes_handle = + scope.spawn(AddressindexToAddressbytes::import); + let addressindex_to_addresstype_handle = scope.spawn(AddressindexToAddresstype::import); + let addressindex_to_txoutindexes_handle = + scope.spawn(AddressindexToTxoutindexes::import); let blockhash_prefix_to_height_handle = scope.spawn(BlockhashPrefixToHeight::import); let height_to_blockhash_handle = scope.spawn(HeightToBlockhash::import); let height_to_first_txindex_handle = @@ -45,16 +67,30 @@ impl Databases { let height_to_last_txindex_handle = scope.spawn(|| HeightToTxindex::import(HeightToTxindexPosition::Last)); let txid_prefix_to_txindex_handle = scope.spawn(TxidPrefixToTxindex::import); + let txindex_to_height_handle = scope.spawn(TxindexToHeight::import); let txindex_to_txid_handle = scope.spawn(TxindexToTxid::import); + let txoutindex_to_addressindex_handle = scope.spawn(TxoutindexToAddressindex::import); let txoutindex_to_amount_handle = scope.spawn(TxoutindexToAmount::import); Ok(Self { + addressbytes_prefix_to_addressindex: addressbytes_prefix_to_addressindex_handle + .join() + .unwrap()?, + addressindex_to_addressbytes: addressindex_to_addressbytes_handle + .join() + .unwrap()?, + addressindex_to_addresstype: addressindex_to_addresstype_handle.join().unwrap()?, + addressindex_to_txoutindexes: addressindex_to_txoutindexes_handle + .join() + .unwrap()?, blockhash_prefix_to_height: blockhash_prefix_to_height_handle.join().unwrap()?, height_to_blockhash: height_to_blockhash_handle.join().unwrap()?, height_to_first_txindex: height_to_first_txindex_handle.join().unwrap()?, height_to_last_txindex: height_to_last_txindex_handle.join().unwrap()?, txid_prefix_to_txindex: txid_prefix_to_txindex_handle.join().unwrap()?, + txindex_to_height: txindex_to_height_handle.join().unwrap()?, txindex_to_txid: txindex_to_txid_handle.join().unwrap()?, + txoutindex_to_addressindex: txoutindex_to_addressindex_handle.join().unwrap()?, txoutindex_to_amount: txoutindex_to_amount_handle.join().unwrap()?, }) }) @@ -62,34 +98,47 @@ impl Databases { pub fn export(&mut self, height: Height) -> color_eyre::Result<()> { thread::scope(|scope| { + scope.spawn(|| { + self.addressbytes_prefix_to_addressindex + .export(height) + .unwrap() + }); + scope.spawn(|| self.addressindex_to_addressbytes.export(height).unwrap()); + scope.spawn(|| self.addressindex_to_addresstype.export(height).unwrap()); + scope.spawn(|| self.addressindex_to_txoutindexes.export(height).unwrap()); scope.spawn(|| self.blockhash_prefix_to_height.export(height).unwrap()); scope.spawn(|| self.height_to_blockhash.export(height).unwrap()); scope.spawn(|| self.height_to_first_txindex.export(height).unwrap()); scope.spawn(|| self.height_to_last_txindex.export(height).unwrap()); scope.spawn(|| self.txid_prefix_to_txindex.export(height).unwrap()); + scope.spawn(|| self.txindex_to_height.export(height).unwrap()); scope.spawn(|| self.txindex_to_txid.export(height).unwrap()); + scope.spawn(|| self.txoutindex_to_addressindex.export(height).unwrap()); scope.spawn(|| self.txoutindex_to_amount.export(height).unwrap()); }); Ok(()) } - pub fn start_height(&self, rpc: &Client) -> color_eyre::Result { - let safe_height = Height::try_from(rpc)?.sub(UNSAFE_BLOCKS); - Ok(self - .min_height() + pub fn start_height(&self) -> Height { + self.min_height() .map(|h| h.sub(UNSAFE_BLOCKS)) .unwrap_or_default() - .min(safe_height)) } fn min_height(&self) -> Option { [ + self.addressbytes_prefix_to_addressindex.height(), + self.addressindex_to_addressbytes.height(), + self.addressindex_to_addresstype.height(), + self.addressindex_to_txoutindexes.height(), self.blockhash_prefix_to_height.height(), self.height_to_blockhash.height(), self.height_to_first_txindex.height(), self.height_to_last_txindex.height(), self.txid_prefix_to_txindex.height(), + self.txindex_to_height.height(), self.txindex_to_txid.height(), + self.txoutindex_to_addressindex.height(), self.txoutindex_to_amount.height(), ] .into_iter() @@ -109,7 +158,11 @@ impl Databases { .is_some_and(|saved_blockhash| blockhash != &saved_blockhash)) } - pub fn erase_from(&mut self, height: Height) -> color_eyre::Result<()> { + pub fn rollback_from(&mut self, height: Height, exit: &Exit) -> color_eyre::Result<()> { + exit.block(); + + self.export(height)?; + let mut txindex = None; self.height_to_blockhash @@ -144,6 +197,7 @@ impl Databases { let txid = Txid::from_slice(&slice_txid)?; self.txindex_to_txid.remove(txindex); + self.txindex_to_height.remove(txindex); self.txid_prefix_to_txindex.remove(&txid); Ok(()) @@ -151,6 +205,8 @@ impl Databases { let txoutindex = Txoutindex::from(txindex); + let mut addressindexes = BTreeSet::new(); + self.txoutindex_to_amount .range(Slice::from(txoutindex)..) .try_for_each(|slice| -> color_eyre::Result<()> { @@ -159,9 +215,42 @@ impl Databases { self.txoutindex_to_amount.remove(txoutindex); + if let Some(addressindex_slice) = + self.txoutindex_to_addressindex.get(txoutindex.into())? + { + self.txoutindex_to_addressindex.remove(txoutindex); + + let addressindex = Addressindex::from(addressindex_slice); + addressindexes.insert(addressindex); + self.addressindex_to_txoutindexes + .remove(addressindex, txoutindex); + } + Ok(()) })?; + self.export(height)?; + + addressindexes + .into_iter() + .filter(|addressindex| self.addressindex_to_txoutindexes.is_empty(*addressindex)) + .try_for_each(|addressindex| -> color_eyre::Result<()> { + let addressbytes = self + .addressindex_to_addressbytes + .get(addressindex)? + .context("addressindex_to_address to have value")?; + self.addressbytes_prefix_to_addressindex + .remove(&addressbytes); + self.addressindex_to_addressbytes.remove(addressindex); + self.addressindex_to_addresstype.remove(addressindex); + + Ok(()) + })?; + + self.export(height)?; + + exit.unblock(); + Ok(()) } } diff --git a/src/crates/barser/src/databases/txid_prefix_to_txindex.rs b/src/crates/bitbase/src/database/txid_prefix_to_txindex.rs similarity index 100% rename from src/crates/barser/src/databases/txid_prefix_to_txindex.rs rename to src/crates/bitbase/src/database/txid_prefix_to_txindex.rs diff --git a/src/crates/bitbase/src/database/txindex_to_height.rs b/src/crates/bitbase/src/database/txindex_to_height.rs new file mode 100644 index 000000000..32bb956a8 --- /dev/null +++ b/src/crates/bitbase/src/database/txindex_to_height.rs @@ -0,0 +1,29 @@ +use derive_deref::{Deref, DerefMut}; + +use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version}; + +#[derive(Deref, DerefMut)] +pub struct TxindexToHeight(Database); + +impl TxindexToHeight { + pub fn import() -> color_eyre::Result { + Ok(Self(Database::import( + "txindex_to_height", + Self::version(), + )?)) + } + + pub fn insert(&mut self, txindex: Txindex, height: Height) { + self.0.insert(txindex.into(), height.into(), height) + } + + pub fn remove(&mut self, txindex: Txindex) { + self.0.remove(txindex.into()) + } +} + +impl DatabaseTrait for TxindexToHeight { + fn version() -> Version { + Version::from(1) + } +} diff --git a/src/crates/barser/src/databases/txindex_to_txid.rs b/src/crates/bitbase/src/database/txindex_to_txid.rs similarity index 91% rename from src/crates/barser/src/databases/txindex_to_txid.rs rename to src/crates/bitbase/src/database/txindex_to_txid.rs index 9c913f98b..7369fd6bf 100644 --- a/src/crates/barser/src/databases/txindex_to_txid.rs +++ b/src/crates/bitbase/src/database/txindex_to_txid.rs @@ -1,6 +1,5 @@ use biter::bitcoin::Txid; use derive_deref::{Deref, DerefMut}; -use fjall::Slice; use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version}; @@ -17,7 +16,7 @@ impl TxindexToTxid { } pub fn remove(&mut self, txindex: Txindex) { - self.0.remove(Slice::from(txindex)) + self.0.remove(txindex.into()) } } diff --git a/src/crates/bitbase/src/database/txoutindex_to_addressindex.rs b/src/crates/bitbase/src/database/txoutindex_to_addressindex.rs new file mode 100644 index 000000000..d71631db7 --- /dev/null +++ b/src/crates/bitbase/src/database/txoutindex_to_addressindex.rs @@ -0,0 +1,30 @@ +use derive_deref::{Deref, DerefMut}; + +use crate::structs::{Addressindex, Database, DatabaseTrait, Height, Txoutindex, Version}; + +#[derive(Deref, DerefMut)] +pub struct TxoutindexToAddressindex(Database); + +impl TxoutindexToAddressindex { + pub fn import() -> color_eyre::Result { + Ok(Self(Database::import( + "txoutindex_to_addressindex", + Self::version(), + )?)) + } + + pub fn insert(&mut self, txoutindex: Txoutindex, addressindex: Addressindex, height: Height) { + self.0 + .insert(txoutindex.into(), addressindex.into(), height) + } + + pub fn remove(&mut self, txoutindex: Txoutindex) { + self.0.remove(txoutindex.into()) + } +} + +impl DatabaseTrait for TxoutindexToAddressindex { + fn version() -> Version { + Version::from(1) + } +} diff --git a/src/crates/barser/src/databases/txoutindex_to_amount.rs b/src/crates/bitbase/src/database/txoutindex_to_amount.rs similarity index 91% rename from src/crates/barser/src/databases/txoutindex_to_amount.rs rename to src/crates/bitbase/src/database/txoutindex_to_amount.rs index 7ac6de829..6a014d342 100644 --- a/src/crates/barser/src/databases/txoutindex_to_amount.rs +++ b/src/crates/bitbase/src/database/txoutindex_to_amount.rs @@ -1,5 +1,4 @@ use derive_deref::{Deref, DerefMut}; -use fjall::Slice; use crate::structs::{Amount, Database, DatabaseTrait, Height, Txoutindex, Version}; @@ -19,7 +18,7 @@ impl TxoutindexToAmount { } pub fn remove(&mut self, txoutindex: Txoutindex) { - self.0.remove(Slice::from(txoutindex)) + self.0.remove(txoutindex.into()) } } diff --git a/src/crates/bitbase/src/main.rs b/src/crates/bitbase/src/main.rs new file mode 100644 index 000000000..5e42ee33d --- /dev/null +++ b/src/crates/bitbase/src/main.rs @@ -0,0 +1,167 @@ +use std::path::Path; + +use biter::bitcoincore_rpc::{Auth, Client}; + +mod database; +mod structs; + +use database::Database; +use fjall::Slice; +use structs::{Addressbytes, Addressindex, Addresstype, Exit, Height, Txindex, Txoutindex}; + +// https://github.com/fjall-rs/fjall/discussions/72 +// https://github.com/romanz/electrs/blob/master/doc/schema.md + +const DAILY_BLOCK_TARGET: usize = 144; +const MONTHLY_BLOCK_TARGET: usize = DAILY_BLOCK_TARGET * 30; + +fn main() -> color_eyre::Result<()> { + let i = std::time::Instant::now(); + + let data_dir = Path::new("../../../../bitcoin"); + let cookie = Path::new(data_dir).join(".cookie"); + let rpc = Client::new("http://localhost:8332", Auth::CookieFile(cookie)).unwrap(); + + let exit = Exit::new(); + + let mut db = Database::import()?; + + let mut height = db.start_height(); + + let mut txindex = db + .height_to_last_txindex + .get(height)? + .map(Txindex::incremented) + .unwrap_or(Txindex::default()); + + let mut addressindex = db + .txoutindex_to_addressindex + .prefix(Slice::from(txindex)) + .last() + .map(|res| -> color_eyre::Result { + Ok(Addressindex::from(res?.1).incremented()) + }) + .unwrap_or(Ok(Addressindex::default()))?; + + let export = |db: &mut Database, height: Height| -> color_eyre::Result<()> { + exit.block(); + println!("Exporting..."); + db.export(height)?; + println!("Export done"); + exit.unblock(); + Ok(()) + }; + + biter::new(data_dir, Some(height.into()), None, rpc) + .iter() + .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { + println!("Processing block {_height}..."); + + height = Height::from(_height); + + if db.has_different_blockhash(height, &blockhash)? { + db.rollback_from(height, &exit)?; + } + + db.blockhash_prefix_to_height.insert(&blockhash, height)?; + db.height_to_blockhash.insert(height, &blockhash); + + let txlen = block.txdata.len(); + let last_txindex = txlen - 1; + + block.txdata.into_iter().enumerate().try_for_each( + |(txi, tx)| -> color_eyre::Result<()> { + if txi == 0 { + db.height_to_first_txindex.insert(height, txindex); + } + if txi == last_txindex { + db.height_to_last_txindex.insert(height, txindex); + } + + if !db.txindex_to_txid.is_safe(height) + || !db.txid_prefix_to_txindex.is_safe(height) + { + let txid = tx.compute_txid(); + db.txindex_to_txid.insert(txindex, &txid, height); + db.txid_prefix_to_txindex.insert(&txid, txindex, height)?; + } + + db.txindex_to_height.insert(txindex, height); + + txindex.increment(); + + tx.output.into_iter().enumerate().try_for_each( + |(vout, txout)| -> color_eyre::Result<()> { + let vout = vout as u16; + let txoutindex = Txoutindex::from((txindex, vout)); + let amount = txout.value.into(); + db.txoutindex_to_amount.insert(txoutindex, amount, height); + + let script = &txout.script_pubkey; + + let addressbytes = + Addressbytes::try_from(script).inspect_err(|_| { + dbg!(&txout, height, txi); + })?; + let addresstype = Addresstype::try_from(script)?; + + let mut addressindex_local = addressindex; + + if let Some(addressindex_slice) = db + .addressbytes_prefix_to_addressindex + .get((&addressbytes).into())? + { + addressindex_local = addressindex_slice.into() + } else { + db.addressbytes_prefix_to_addressindex + .insert(&addressbytes, addressindex_local, height) + .inspect_err(|_| { + dbg!(addresstype); + })?; + db.addressindex_to_addressbytes.insert( + addressindex_local, + &addressbytes, + height, + ); + db.addressindex_to_addresstype.insert( + addressindex_local, + addresstype, + height, + ); + addressindex.increment(); + } + + db.txoutindex_to_addressindex.insert( + txoutindex, + addressindex_local, + height, + ); + + db.addressindex_to_txoutindexes.insert( + addressindex_local, + txoutindex, + height, + ); + + Ok(()) + }, + )?; + + Ok(()) + }, + )?; + + let should_snapshot = _height % MONTHLY_BLOCK_TARGET == 0 && !exit.active(); + if should_snapshot { + export(&mut db, height)?; + } + + Ok(()) + })?; + + export(&mut db, height)?; + + dbg!(i.elapsed()); + + Ok(()) +} diff --git a/src/crates/bitbase/src/structs/addressbytes.rs b/src/crates/bitbase/src/structs/addressbytes.rs new file mode 100644 index 000000000..f3533abd9 --- /dev/null +++ b/src/crates/bitbase/src/structs/addressbytes.rs @@ -0,0 +1,53 @@ +use biter::bitcoin::ScriptBuf; +use color_eyre::eyre::eyre; +use fjall::Slice; + +#[derive(Debug)] +pub struct Addressbytes(Slice); + +impl Addressbytes { + pub fn to_prefix_slice(&self) -> Slice { + self.0[..8].into() + } +} + +impl TryFrom<&ScriptBuf> for Addressbytes { + type Error = color_eyre::Report; + fn try_from(script: &ScriptBuf) -> Result { + if script.is_p2pk() { + let bytes = script.as_bytes(); + let bytes = match bytes.len() { + 67 => &script.as_bytes()[1..66], + 35 => &script.as_bytes()[1..34], + _ => { + dbg!(bytes); + return Err(eyre!("Wrong len")); + } + }; + + if bytes[0] != 4 { + dbg!(bytes); + return Err(eyre!("Doesn't start with a 4")); + } + + Ok(Self(bytes.into())) + } else if script.is_p2pkh() { + let bytes = &script.as_bytes()[3..23]; + + Ok(Self(bytes.into())) + } else { + Err(eyre!("Unsupported address type")) + } + } +} + +impl From for Addressbytes { + fn from(value: Slice) -> Self { + Self(value) + } +} +impl From<&Addressbytes> for Slice { + fn from(value: &Addressbytes) -> Self { + value.0.clone() + } +} diff --git a/src/crates/bitbase/src/structs/addressindex.rs b/src/crates/bitbase/src/structs/addressindex.rs new file mode 100644 index 000000000..0d53e05d2 --- /dev/null +++ b/src/crates/bitbase/src/structs/addressindex.rs @@ -0,0 +1,47 @@ +use derive_deref::{Deref, DerefMut}; +use fjall::Slice; + +use super::SliceExtended; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +pub struct Addressindex(u32); + +impl Addressindex { + pub const BYTES: usize = size_of::(); + + pub fn increment(&mut self) { + self.0 += 1; + } + + pub fn incremented(self) -> Self { + Self(*self + 1) + } +} + +impl From for Addressindex { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for Addressindex { + fn from(value: u64) -> Self { + Self(value as u32) + } +} +impl From for u64 { + fn from(value: Addressindex) -> Self { + value.0 as u64 + } +} + +impl From for Addressindex { + fn from(slice: Slice) -> Self { + Self(slice.read_u32()) + } +} +impl From for Slice { + fn from(value: Addressindex) -> Self { + value.to_be_bytes().into() + } +} diff --git a/src/crates/bitbase/src/structs/addresstxoutindex.rs b/src/crates/bitbase/src/structs/addresstxoutindex.rs new file mode 100644 index 000000000..b83cde83e --- /dev/null +++ b/src/crates/bitbase/src/structs/addresstxoutindex.rs @@ -0,0 +1,35 @@ +use fjall::Slice; + +use super::{Addressindex, Txoutindex}; + +pub struct Addresstxoutindex { + addressindex: Addressindex, + txoutindex: Txoutindex, +} + +impl From<(Addressindex, Txoutindex)> for Addresstxoutindex { + fn from(value: (Addressindex, Txoutindex)) -> Self { + Self { + addressindex: value.0, + txoutindex: value.1, + } + } +} + +impl From for Slice { + fn from(value: Addresstxoutindex) -> Self { + let txindex_slice = Self::from(value.addressindex); + let vout_slice = Self::from(value.txoutindex); + Self::from([txindex_slice, vout_slice].concat()) + } +} +impl From for Addresstxoutindex { + fn from(value: Slice) -> Self { + let addressindex = Addressindex::from(Slice::from(&value[..Addressindex::BYTES])); + let txoutindex = Txoutindex::from(Slice::from(&value[Addressindex::BYTES..])); + Self { + addressindex, + txoutindex, + } + } +} diff --git a/src/crates/bitbase/src/structs/addresstype.rs b/src/crates/bitbase/src/structs/addresstype.rs new file mode 100644 index 000000000..716c844d1 --- /dev/null +++ b/src/crates/bitbase/src/structs/addresstype.rs @@ -0,0 +1,40 @@ +use biter::bitcoin::ScriptBuf; +use color_eyre::eyre::eyre; +use fjall::Slice; + +use super::SliceExtended; + +#[derive(Debug, Clone, Copy)] +pub enum Addresstype { + P2PK, + P2PKH, +} + +impl TryFrom<&ScriptBuf> for Addresstype { + type Error = color_eyre::Report; + fn try_from(value: &ScriptBuf) -> Result { + if value.is_p2pk() { + Ok(Self::P2PK) + } else if value.is_p2pkh() { + Ok(Self::P2PKH) + } else { + Err(eyre!("Not compatible script")) + } + } +} + +impl TryFrom for Addresstype { + type Error = color_eyre::Report; + fn try_from(value: Slice) -> Result { + match value.read_u8() { + x if x == Addresstype::P2PK as u8 => Ok(Addresstype::P2PK), + x if x == Addresstype::P2PKH as u8 => Ok(Addresstype::P2PKH), + _ => Err(eyre!("Unknown type")), + } + } +} +impl From for Slice { + fn from(addresstype: Addresstype) -> Self { + (addresstype as u8).to_be_bytes().into() + } +} diff --git a/src/crates/barser/src/structs/amount.rs b/src/crates/bitbase/src/structs/amount.rs similarity index 97% rename from src/crates/barser/src/structs/amount.rs rename to src/crates/bitbase/src/structs/amount.rs index d25acf47a..0219842d7 100644 --- a/src/crates/barser/src/structs/amount.rs +++ b/src/crates/bitbase/src/structs/amount.rs @@ -5,6 +5,7 @@ use std::{ use biter::bitcoin; use derive_deref::{Deref, DerefMut}; +use fjall::Slice; use super::Height; @@ -29,7 +30,7 @@ impl From for Amount { } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: Amount) -> Self { value.to_sat().to_be_bytes().into() } diff --git a/src/crates/barser/src/structs/exit.rs b/src/crates/bitbase/src/structs/exit.rs similarity index 100% rename from src/crates/barser/src/structs/exit.rs rename to src/crates/bitbase/src/structs/exit.rs diff --git a/src/crates/barser/src/structs/database.rs b/src/crates/bitbase/src/structs/fjall.rs similarity index 90% rename from src/crates/barser/src/structs/database.rs rename to src/crates/bitbase/src/structs/fjall.rs index f33aa304f..cfae90a45 100644 --- a/src/crates/barser/src/structs/database.rs +++ b/src/crates/bitbase/src/structs/fjall.rs @@ -1,10 +1,8 @@ use std::{mem, ops::RangeBounds}; -use fjall::{ - Batch, Keyspace, KvPair, PartitionCreateOptions, PartitionHandle, PersistMode, Result, Slice, -}; +pub use fjall::*; -use super::{Height, Version}; +use crate::structs::{Height, Version}; pub struct Database { keyspace: Keyspace, @@ -19,7 +17,7 @@ const HEIGHT: &str = "height"; impl Database { pub fn import(name: &str, version: Version) -> Result { - let keyspace = fjall::Config::new(format!("./databases/{name}")).open()?; + let keyspace = fjall::Config::new(format!("./database/{name}")).open()?; let data = Self::open_data(&keyspace)?; let meta = Self::open_meta(&keyspace)?; @@ -69,6 +67,13 @@ impl Database { self.data.range(range) } + pub fn prefix<'a, K: AsRef<[u8]> + 'a>( + &'a self, + prefix: K, + ) -> impl DoubleEndedIterator> + 'static { + self.data.prefix(prefix) + } + pub fn insert(&mut self, key: Slice, value: Slice, height: Height) { if self.is_safe(height) { return; diff --git a/src/crates/barser/src/structs/height.rs b/src/crates/bitbase/src/structs/height.rs similarity index 94% rename from src/crates/barser/src/structs/height.rs rename to src/crates/bitbase/src/structs/height.rs index d39992459..cd5aa23fb 100644 --- a/src/crates/barser/src/structs/height.rs +++ b/src/crates/bitbase/src/structs/height.rs @@ -5,18 +5,19 @@ use std::{ use biter::bitcoincore_rpc::{self, RpcApi}; use derive_deref::{Deref, DerefMut}; +use fjall::Slice; use super::SliceExtended; #[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct Height(u32); -impl From for Height { - fn from(slice: fjall::Slice) -> Self { +impl From for Height { + fn from(slice: Slice) -> Self { Self(slice.read_u32()) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: Height) -> Self { value.to_be_bytes().into() } diff --git a/src/crates/barser/src/structs/mod.rs b/src/crates/bitbase/src/structs/mod.rs similarity index 51% rename from src/crates/barser/src/structs/mod.rs rename to src/crates/bitbase/src/structs/mod.rs index 574b6655a..ad6ef9954 100644 --- a/src/crates/barser/src/structs/mod.rs +++ b/src/crates/bitbase/src/structs/mod.rs @@ -1,15 +1,23 @@ +mod addressbytes; +mod addressindex; +mod addresstxoutindex; +mod addresstype; mod amount; -mod database; mod exit; +mod fjall; mod height; mod slice; mod txindex; mod txoutindex; mod version; +pub use addressbytes::*; +pub use addressindex::*; +pub use addresstxoutindex::*; +pub use addresstype::*; pub use amount::*; -pub use database::*; pub use exit::*; +pub use fjall::*; pub use height::*; pub use slice::*; pub use txindex::*; diff --git a/src/crates/barser/src/structs/slice.rs b/src/crates/bitbase/src/structs/slice.rs similarity index 70% rename from src/crates/barser/src/structs/slice.rs rename to src/crates/bitbase/src/structs/slice.rs index 2602f8bd5..98e47c7f3 100644 --- a/src/crates/barser/src/structs/slice.rs +++ b/src/crates/bitbase/src/structs/slice.rs @@ -1,19 +1,34 @@ use std::io::Read; +use fjall::Slice; + +#[allow(unused)] pub trait SliceExtended { + fn default() -> Self; fn read_u8(&self) -> u8; + fn read_u16(&self) -> u16; fn read_u32(&self) -> u32; fn read_u64(&self) -> u64; fn read_exact(&self, buf: &mut [u8]); } -impl SliceExtended for fjall::Slice { +impl SliceExtended for Slice { + fn default() -> Self { + Self::new(&[]) + } + fn read_u8(&self) -> u8 { let mut buf: [u8; 1] = [0; 1]; self.read_exact(&mut buf); u8::from_be_bytes(buf) } + fn read_u16(&self) -> u16 { + let mut buf: [u8; 2] = [0; 2]; + self.read_exact(&mut buf); + u16::from_be_bytes(buf) + } + fn read_u32(&self) -> u32 { let mut buf: [u8; 4] = [0; 4]; self.read_exact(&mut buf); diff --git a/src/crates/barser/src/structs/txindex.rs b/src/crates/bitbase/src/structs/txindex.rs similarity index 81% rename from src/crates/barser/src/structs/txindex.rs rename to src/crates/bitbase/src/structs/txindex.rs index 45d02e751..2df5434fa 100644 --- a/src/crates/barser/src/structs/txindex.rs +++ b/src/crates/bitbase/src/structs/txindex.rs @@ -1,4 +1,5 @@ use derive_deref::{Deref, DerefMut}; +use fjall::Slice; use super::SliceExtended; @@ -6,6 +7,8 @@ use super::SliceExtended; pub struct Txindex(u32); impl Txindex { + pub const BYTES: usize = size_of::(); + pub fn increment(&mut self) { self.0 += 1; } @@ -32,12 +35,12 @@ impl From for u64 { } } -impl From for Txindex { - fn from(slice: fjall::Slice) -> Self { +impl From for Txindex { + fn from(slice: Slice) -> Self { Self(slice.read_u32()) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: Txindex) -> Self { value.to_be_bytes().into() } diff --git a/src/crates/bitbase/src/structs/txoutindex.rs b/src/crates/bitbase/src/structs/txoutindex.rs new file mode 100644 index 000000000..69b194aff --- /dev/null +++ b/src/crates/bitbase/src/structs/txoutindex.rs @@ -0,0 +1,46 @@ +use fjall::Slice; + +use super::{SliceExtended, Txindex}; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)] +pub struct Txoutindex { + pub txindex: Txindex, + pub vout: u16, +} + +impl Txoutindex { + pub const BYTES: usize = size_of::(); +} + +impl From for Txoutindex { + fn from(value: Txindex) -> Self { + Self { + txindex: value, + vout: 0, + } + } +} + +impl From<(Txindex, u16)> for Txoutindex { + fn from(value: (Txindex, u16)) -> Self { + Self { + txindex: value.0, + vout: value.1, + } + } +} + +impl From for Slice { + fn from(value: Txoutindex) -> Self { + let txindex_slice = Self::from(value.txindex); + let vout_slice = Self::from(value.vout.to_be_bytes()); + Self::from([txindex_slice, vout_slice].concat()) + } +} +impl From for Txoutindex { + fn from(value: Slice) -> Self { + let txindex = Txindex::from(Slice::from(&value[..Txindex::BYTES])); + let vout = Slice::from(&value[Txindex::BYTES..]).read_u16(); + Self { txindex, vout } + } +} diff --git a/src/crates/barser/src/structs/version.rs b/src/crates/bitbase/src/structs/version.rs similarity index 77% rename from src/crates/barser/src/structs/version.rs rename to src/crates/bitbase/src/structs/version.rs index 4c0dfe27e..0c4692388 100644 --- a/src/crates/barser/src/structs/version.rs +++ b/src/crates/bitbase/src/structs/version.rs @@ -1,4 +1,5 @@ use derive_deref::{Deref, DerefMut}; +use fjall::Slice; use super::SliceExtended; @@ -11,8 +12,8 @@ impl From for Version { } } -impl From for Version { - fn from(slice: fjall::Slice) -> Self { +impl From for Version { + fn from(slice: Slice) -> Self { Self(slice.read_u8()) } } diff --git a/src/crates/biter/Cargo.lock b/src/crates/biter/Cargo.lock index 0de88bd25..04c5c6ee7 100644 --- a/src/crates/biter/Cargo.lock +++ b/src/crates/biter/Cargo.lock @@ -243,9 +243,9 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jsonrpc" @@ -308,9 +308,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -394,18 +394,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -414,9 +414,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -432,9 +432,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "syn" -version = "2.0.90" +version = "2.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "9c786062daee0d6db1132800e623df74274a0a87322d8e183338e01b3d98d058" dependencies = [ "proc-macro2", "quote", diff --git a/src/crates/biter/Cargo.toml b/src/crates/biter/Cargo.toml index e9ad3f63a..26db5dd47 100644 --- a/src/crates/biter/Cargo.toml +++ b/src/crates/biter/Cargo.toml @@ -12,8 +12,8 @@ edition = "2021" bitcoin = { version = "0.32.5", features = ["serde"] } rayon = "1.10.0" crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } -serde = { version = "1.0.216", features = ["derive"] } -serde_json = "1.0.133" +serde = { version = "1.0.217", features = ["derive"] } +serde_json = "1.0.134" derived-deref = "2.1.0" bitcoincore-rpc = "0.19.0" # tokio = { version = "1.39.2", features = ["rt-multi-thread"] } diff --git a/src/crates/biter/src/blk_index_to_blk_path.rs b/src/crates/biter/src/blk_index_to_blk_path.rs new file mode 100644 index 000000000..17cf2da39 --- /dev/null +++ b/src/crates/biter/src/blk_index_to_blk_path.rs @@ -0,0 +1,46 @@ +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +use derived_deref::{Deref, DerefMut}; + +const BLK: &str = "blk"; +const DAT: &str = ".dat"; + +#[derive(Debug, Deref, DerefMut)] +pub struct BlkIndexToBlkPath(BTreeMap); + +impl BlkIndexToBlkPath { + pub fn scan(data_dir: &Path) -> Self { + let blocks_dir = data_dir.join("blocks"); + + Self( + fs::read_dir(blocks_dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + let is_file = path.is_file(); + + if is_file { + let file_name = path.file_name().unwrap().to_str().unwrap(); + + file_name.starts_with(BLK) && file_name.ends_with(DAT) + } else { + false + } + }) + .map(|path| { + let file_name = path.file_name().unwrap().to_str().unwrap(); + + let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())] + .parse::() + .unwrap(); + + (blk_index, path) + }) + .collect::>(), + ) + } +} diff --git a/src/crates/biter/src/blk_index_to_blk_recap.rs b/src/crates/biter/src/blk_index_to_blk_recap.rs index b0353261f..b819448a6 100644 --- a/src/crates/biter/src/blk_index_to_blk_recap.rs +++ b/src/crates/biter/src/blk_index_to_blk_recap.rs @@ -8,7 +8,7 @@ use std::{ use derived_deref::{Deref, DerefMut}; -use crate::{blk_recap::BlkRecap, BlkMetadataAndBlock}; +use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock}; const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30; @@ -21,7 +21,7 @@ pub struct BlkIndexToBlkRecap { } impl BlkIndexToBlkRecap { - pub fn import(blocks_dir: &BTreeMap, data_dir: &Path) -> Self { + pub fn import(blocks_dir: &BlkIndexToBlkPath, data_dir: &Path) -> Self { let path = data_dir.join("blk_index_to_blk_recap.json"); let tree = { @@ -46,7 +46,7 @@ impl BlkIndexToBlkRecap { this } - pub fn clean_outdated(&mut self, blocks_dir: &BTreeMap) { + pub fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) { let mut unprocessed_keys = self.keys().copied().collect::>(); blocks_dir.iter().for_each(|(blk_index, blk_path)| { diff --git a/src/crates/biter/src/lib.rs b/src/crates/biter/src/lib.rs index ed80f1882..55daa343e 100644 --- a/src/crates/biter/src/lib.rs +++ b/src/crates/biter/src/lib.rs @@ -13,12 +13,14 @@ use bitcoin::{ Block, BlockHash, }; use bitcoincore_rpc::RpcApi; +use blk_index_to_blk_path::*; use crossbeam::channel::{bounded, Receiver}; use rayon::prelude::*; pub use bitcoin; pub use bitcoincore_rpc; +mod blk_index_to_blk_path; mod blk_index_to_blk_recap; mod blk_metadata; mod blk_metadata_and_block; @@ -88,7 +90,7 @@ pub fn new( let (send_block, recv_block) = bounded(BOUND_CAP); let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP); - let blk_index_to_blk_path = scan_blocks_dir(data_dir); + let blk_index_to_blk_path = BlkIndexToBlkPath::scan(data_dir); let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blk_index_to_blk_path, data_dir); @@ -97,12 +99,12 @@ pub fn new( thread::spawn(move || { blk_index_to_blk_path - .into_iter() - .filter(|(blk_index, _)| blk_index >= &starting_blk_index) + .iter() + .filter(|(blk_index, _)| blk_index >= &&starting_blk_index) .try_for_each(move |(blk_index, blk_path)| { - let blk_metadata = BlkMetadata::new(blk_index, &blk_path); + let blk_metadata = BlkMetadata::new(*blk_index, blk_path); - let blk_bytes = fs::read(&blk_path).unwrap(); + let blk_bytes = fs::read(blk_path).unwrap(); let blk_bytes_len = blk_bytes.len() as u64; let mut cursor = Cursor::new(blk_bytes.as_slice()); diff --git a/src/crates/biter/src/utils.rs b/src/crates/biter/src/utils.rs index 78819179e..0894cd24d 100644 --- a/src/crates/biter/src/utils.rs +++ b/src/crates/biter/src/utils.rs @@ -1,41 +1,4 @@ -use std::{ - collections::BTreeMap, - fs, - path::{Path, PathBuf}, - time::UNIX_EPOCH, -}; - -const BLK: &str = "blk"; -const DAT: &str = ".dat"; - -pub fn scan_blocks_dir(data_dir: &Path) -> BTreeMap { - let blocks_dir = data_dir.join("blocks"); - - fs::read_dir(blocks_dir) - .unwrap() - .map(|entry| entry.unwrap().path()) - .filter(|path| { - let is_file = path.is_file(); - - if is_file { - let file_name = path.file_name().unwrap().to_str().unwrap(); - - file_name.starts_with(BLK) && file_name.ends_with(DAT) - } else { - false - } - }) - .map(|path| { - let file_name = path.file_name().unwrap().to_str().unwrap(); - - let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())] - .parse::() - .unwrap(); - - (blk_index, path) - }) - .collect::>() -} +use std::{fs, path::PathBuf, time::UNIX_EPOCH}; pub fn path_to_modified_time(path: &PathBuf) -> u64 { fs::metadata(path) diff --git a/src/parser/actions/iter_blocks.rs b/src/parser/actions/iter_blocks.rs index de9b630b2..ba79b13d9 100644 --- a/src/parser/actions/iter_blocks.rs +++ b/src/parser/actions/iter_blocks.rs @@ -21,7 +21,7 @@ use crate::{ pub fn iter_blocks( config: &Config, - rpc: &biter::bitcoincore_rpc::Client, + rpc: &Client, approx_block_count: usize, exit: Exit, databases: &mut Databases,