diff --git a/parser/Cargo.lock b/parser/Cargo.lock index f30b0ec99..5a8466a1e 100644 --- a/parser/Cargo.lock +++ b/parser/Cargo.lock @@ -170,6 +170,15 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bincode" version = "2.0.0-rc.3" @@ -292,6 +301,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "bumpalo" @@ -319,13 +331,12 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.99" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" dependencies = [ "jobserver", "libc", - "once_cell", ] [[package]] @@ -540,6 +551,15 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + [[package]] name = "either" version = "1.12.0" @@ -744,6 +764,44 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "heed" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc30da4a93ff8cb98e535d595d6de42731d4719d707bc1c86f579158751a24e" +dependencies = [ + "bitflags 2.6.0", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" +dependencies = [ + "bincode 1.3.3", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1045,6 +1103,17 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lmdb-master-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57640c190703d5ccf4a86aff4aeb749b2d287a8cb1723c76b51f39d77ab53b24" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -1235,6 +1304,16 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1278,13 +1357,14 @@ name = "parser" version = "0.3.0" dependencies = [ "allocative", - "bincode", + "bincode 2.0.0-rc.3", "bitcoin_hashes", "biter", "chrono", "clap", "color-eyre", "derive_deref", + "heed", "inferno", "itertools", "memory-stats", @@ -1304,6 +1384,48 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -1721,6 +1843,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -1798,6 +1926,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -2071,9 +2208,9 @@ checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 2df89367b..e6bf6d940 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -28,3 +28,4 @@ serde = { version = "1.0.204", features = ["derive"] } serde_json = "1.0.122" toml = "0.8.19" zstd = "0.13.2" +heed = "0.20.3" diff --git a/parser/src/actions/parse.rs b/parser/src/actions/parse.rs index fa335e026..9a94e4fd9 100644 --- a/parser/src/actions/parse.rs +++ b/parser/src/actions/parse.rs @@ -129,6 +129,7 @@ pub fn parse( let mut txouts_parsing_results = pre_process_outputs( &block, compute_addresses, + &mut states.address_counters.multisig_addresses, &mut states.address_counters.op_return_addresses, &mut states.address_counters.push_only_addresses, &mut states.address_counters.unknown_addresses, @@ -341,8 +342,10 @@ pub fn parse( if input_tx_data.is_none() { if !enable_check_if_txout_value_is_zero_in_db || rpc - .get_tx_out(&input_txid, input_vout, None) + .get_raw_transaction(&input_txid, None) .unwrap() + .output + .get(input_vout as usize) .unwrap() .value .to_sat() @@ -774,9 +777,11 @@ pub struct TxoutsParsingResults { op_returns: usize, } +#[allow(clippy::too_many_arguments)] fn pre_process_outputs( block: &Block, compute_addresses: bool, + multisig_addresses: &mut Counter, op_return_addresses: &mut Counter, push_only_addresses: &mut Counter, unknown_addresses: &mut Counter, @@ -821,6 +826,7 @@ fn pre_process_outputs( let address_opt = compute_addresses.then(|| { let address = Address::from( txout, + multisig_addresses, op_return_addresses, push_only_addresses, unknown_addresses, diff --git a/parser/src/databases/_database.rs b/parser/src/databases/_database.rs index fdc0fb33b..43e82d439 100644 --- a/parser/src/databases/_database.rs +++ b/parser/src/databases/_database.rs @@ -15,53 +15,37 @@ use derive_deref::{Deref, DerefMut}; // // Possible compression: https://pijul.org/posts/sanakirja-zstd/ use sanakirja::{ - btree::{self, page, page_unsized, BTreeMutPage, Db_}, + btree::{self, page, Db_}, direct_repr, Commit, Env, Error, MutTxn, RootDb, Storable, UnsizedStorable, }; use crate::io::OUTPUTS_FOLDER_PATH; -pub type SizedDatabase = Database>; - -pub type UnsizedDatabase = - Database>; - #[derive(Allocative)] -#[allocative(bound = "KeyTree: Allocative, KeyDB, Value: Allocative, Page")] +#[allocative(bound = "Key: Allocative, Value: Allocative")] /// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way. -pub struct Database +pub struct Database where - KeyTree: Ord + Clone + Debug, - KeyDB: Ord + ?Sized + Storable, + Key: Ord + Clone + Debug + ?Sized + Storable, Value: Storable + PartialEq, - Page: BTreeMutPage, { - pub cached_puts: BTreeMap, - pub cached_dels: BTreeSet, + pub cached_puts: BTreeMap, + pub cached_dels: BTreeSet, #[allocative(skip)] - db: Db_, + db: Db_>, #[allocative(skip)] txn: MutTxn, - #[allocative(skip)] - key_tree_to_key_db: fn(&KeyTree) -> &KeyDB, } -pub const SANAKIRJA_MAX_KEY_SIZE: usize = 510; const ROOT_DB: usize = 0; const PAGE_SIZE: u64 = 4096 * 256; // 1mo - Must be a multiplier of 4096 -impl Database +impl Database where - KeyTree: Ord + Clone + Debug, - KeyDB: Ord + ?Sized + Storable, + Key: Ord + Clone + Debug + ?Sized + Storable, Value: Storable + PartialEq, - Page: BTreeMutPage, { - pub fn open( - folder: &str, - file: &str, - key_tree_to_key_db: fn(&KeyTree) -> &KeyDB, - ) -> color_eyre::Result { + pub fn open(folder: &str, file: &str) -> color_eyre::Result { let mut txn = Self::init_txn(folder, file)?; let db = txn @@ -73,20 +57,19 @@ where cached_dels: BTreeSet::default(), db, txn, - key_tree_to_key_db, }) } pub fn iter(&self, callback: &mut F) where - F: FnMut((&KeyDB, &Value)), + F: FnMut((&Key, &Value)), { btree::iter(&self.txn, &self.db, None) .unwrap() .for_each(|entry| callback(entry.unwrap())); } - pub fn get(&self, key: &KeyTree) -> Option<&Value> { + pub fn get(&self, key: &Key) -> Option<&Value> { if let Some(cached_put) = self.get_from_puts(key) { return Some(cached_put); } @@ -94,13 +77,11 @@ where self.db_get(key) } - pub fn db_get(&self, key: &KeyTree) -> Option<&Value> { - let k = (self.key_tree_to_key_db)(key); + pub fn db_get(&self, key: &Key) -> Option<&Value> { + let option = btree::get(&self.txn, &self.db, key, None).unwrap(); - let option = btree::get(&self.txn, &self.db, k, None).unwrap(); - - if let Some((k_found, v)) = option { - if k == k_found { + if let Some((key_found, v)) = option { + if key == key_found { return Some(v); } } @@ -109,17 +90,17 @@ where } #[inline(always)] - pub fn get_from_puts(&self, key: &KeyTree) -> Option<&Value> { + pub fn get_from_puts(&self, key: &Key) -> Option<&Value> { self.cached_puts.get(key) } #[inline(always)] - pub fn get_mut_from_puts(&mut self, key: &KeyTree) -> Option<&mut Value> { + pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> { self.cached_puts.get_mut(key) } #[inline(always)] - pub fn remove(&mut self, key: &KeyTree) -> Option { + pub fn remove(&mut self, key: &Key) -> Option { self.remove_from_puts(key).or_else(|| { self.db_remove(key); @@ -128,30 +109,30 @@ where } #[inline(always)] - pub fn db_remove(&mut self, key: &KeyTree) { + pub fn db_remove(&mut self, key: &Key) { self.cached_dels.insert(key.clone()); } - pub fn update(&mut self, key: KeyTree, value: Value) -> Option { + pub fn update(&mut self, key: Key, value: Value) -> Option { self.cached_dels.insert(key.clone()); self.cached_puts.insert(key, value) } #[inline(always)] - pub fn remove_from_puts(&mut self, key: &KeyTree) -> Option { + pub fn remove_from_puts(&mut self, key: &Key) -> Option { self.cached_puts.remove(key) } #[inline(always)] - pub fn insert(&mut self, key: KeyTree, value: Value) -> Option { + pub fn insert(&mut self, key: Key, value: Value) -> Option { self.cached_dels.remove(&key); self.unsafe_insert(key, value) } #[inline(always)] - pub fn unsafe_insert(&mut self, key: KeyTree, value: Value) -> Option { + pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { self.cached_puts.insert(key, value) } @@ -175,12 +156,7 @@ where self.cached_dels .into_iter() .try_for_each(|key| -> Result<(), Error> { - btree::del( - &mut self.txn, - &mut self.db, - (self.key_tree_to_key_db)(&key), - None, - )?; + btree::del(&mut self.txn, &mut self.db, &key, None)?; Ok(()) })?; @@ -188,12 +164,7 @@ where self.cached_puts .into_iter() .try_for_each(|(key, value)| -> Result<(), Error> { - btree::put( - &mut self.txn, - &mut self.db, - (self.key_tree_to_key_db)(&key), - &value, - )?; + btree::put(&mut self.txn, &mut self.db, &key, &value)?; Ok(()) })?; diff --git a/parser/src/databases/address_index_to_address_data.rs b/parser/src/databases/address_index_to_address_data.rs index c2422d21b..fb474c1f8 100644 --- a/parser/src/databases/address_index_to_address_data.rs +++ b/parser/src/databases/address_index_to_address_data.rs @@ -12,11 +12,11 @@ use crate::{ utils::time, }; -use super::{AnyDatabaseGroup, Metadata, SizedDatabase}; +use super::{AnyDatabaseGroup, Database as _Database, Metadata}; type Key = u32; type Value = AddressData; -type Database = SizedDatabase; +type Database = _Database; #[derive(Allocative)] pub struct AddressIndexToAddressData { @@ -78,7 +78,7 @@ impl AddressIndexToAddressData { (db_index + 1) * DB_MAX_SIZE ); - SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap() + Database::open(Self::folder(), &db_name).unwrap() }) } diff --git a/parser/src/databases/address_index_to_empty_address_data.rs b/parser/src/databases/address_index_to_empty_address_data.rs index 9e2e89c64..5e803c775 100644 --- a/parser/src/databases/address_index_to_empty_address_data.rs +++ b/parser/src/databases/address_index_to_empty_address_data.rs @@ -9,11 +9,11 @@ use rayon::prelude::*; use crate::structs::{Date, EmptyAddressData, Height}; -use super::{AnyDatabaseGroup, Metadata, SizedDatabase}; +use super::{AnyDatabaseGroup, Database as _Database, Metadata}; type Key = u32; type Value = EmptyAddressData; -type Database = SizedDatabase; +type Database = _Database; #[derive(Allocative)] pub struct AddressIndexToEmptyAddressData { @@ -45,12 +45,6 @@ impl AddressIndexToEmptyAddressData { self.open_db(&key).unsafe_insert(key, value) } - // pub fn undo_insert(&mut self, key: &Key) -> Option { - // self.metadata.called_remove(); - - // self.open_db(key).remove_from_puts(key) - // } - pub fn remove(&mut self, key: &Key) -> Option { self.metadata.called_remove(); @@ -86,7 +80,7 @@ impl AddressIndexToEmptyAddressData { (db_index + 1) * DB_MAX_SIZE ); - SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap() + Database::open(Self::folder(), &db_name).unwrap() }) } diff --git a/parser/src/databases/address_to_address_index.rs b/parser/src/databases/address_to_address_index.rs index d63170701..712fa0837 100644 --- a/parser/src/databases/address_to_address_index.rs +++ b/parser/src/databases/address_to_address_index.rs @@ -5,16 +5,12 @@ use rayon::prelude::*; use crate::structs::{Address, Date, Height}; -use super::{ - AnyDatabaseGroup, Database, Metadata, SizedDatabase, U8x19, U8x31, - UnsizedDatabase as _UnsizedDatabase, -}; +use super::{AnyDatabaseGroup, Database, Metadata, U8x19, U8x31}; type Value = u32; -type U8x19Database = SizedDatabase; -type U8x31Database = SizedDatabase; -type U32Database = SizedDatabase; -type UnsizedDatabase = _UnsizedDatabase, [u8], Value>; +type U8x19Database = Database; +type U8x31Database = Database; +type U32Database = Database; type P2PKDatabase = U8x19Database; type P2PKHDatabase = U8x19Database; @@ -26,7 +22,7 @@ type UnknownDatabase = U32Database; type OpReturnDatabase = U32Database; type PushOnlyDatabase = U32Database; type EmptyDatabase = U32Database; -type MultisigDatabase = UnsizedDatabase; +type MultisigDatabase = U32Database; #[derive(Allocative)] pub struct AddressToAddressIndex { @@ -106,7 +102,7 @@ impl AddressToAddressIndex { Address::Unknown(key) => self.unknown.as_ref().unwrap().get(key), Address::OpReturn(key) => self.op_return.as_ref().unwrap().get(key), Address::PushOnly(key) => self.push_only.as_ref().unwrap().get(key), - Address::MultiSig(key) => self.multisig.as_ref().unwrap().get(key), + Address::MultiSig(key) => self.push_only.as_ref().unwrap().get(key), Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get(key), Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get(key), Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get(key), @@ -155,7 +151,6 @@ impl AddressToAddressIndex { Database::open( &format!("{}/{}", Self::folder(), "p2pk"), &prefix.to_string(), - |key| key, ) .unwrap() }) @@ -166,7 +161,6 @@ impl AddressToAddressIndex { Database::open( &format!("{}/{}", Self::folder(), "p2pkh"), &prefix.to_string(), - |key| key, ) .unwrap() }) @@ -177,7 +171,6 @@ impl AddressToAddressIndex { Database::open( &format!("{}/{}", Self::folder(), "p2sh"), &prefix.to_string(), - |key| key, ) .unwrap() }) @@ -188,7 +181,6 @@ impl AddressToAddressIndex { Database::open( &format!("{}/{}", Self::folder(), "p2wpkh"), &prefix.to_string(), - |key| key, ) .unwrap() }) @@ -199,7 +191,6 @@ impl AddressToAddressIndex { Database::open( &format!("{}/{}", Self::folder(), "p2wsh"), &prefix.to_string(), - |key| key, ) .unwrap() }) @@ -210,7 +201,6 @@ impl AddressToAddressIndex { Database::open( &format!("{}/{}", Self::folder(), "p2tr"), &prefix.to_string(), - |key| key, ) .unwrap() }) @@ -218,28 +208,27 @@ impl AddressToAddressIndex { pub fn open_unknown(&mut self) -> &mut UnknownDatabase { self.unknown - .get_or_insert_with(|| Database::open(Self::folder(), "unknown", |key| key).unwrap()) + .get_or_insert_with(|| Database::open(Self::folder(), "unknown").unwrap()) } pub fn open_op_return(&mut self) -> &mut UnknownDatabase { self.op_return - .get_or_insert_with(|| Database::open(Self::folder(), "op_return", |key| key).unwrap()) + .get_or_insert_with(|| Database::open(Self::folder(), "op_return").unwrap()) } pub fn open_push_only(&mut self) -> &mut UnknownDatabase { self.push_only - .get_or_insert_with(|| Database::open(Self::folder(), "push_only", |key| key).unwrap()) + .get_or_insert_with(|| Database::open(Self::folder(), "push_only").unwrap()) } pub fn open_empty(&mut self) -> &mut UnknownDatabase { self.empty - .get_or_insert_with(|| Database::open(Self::folder(), "empty", |key| key).unwrap()) + .get_or_insert_with(|| Database::open(Self::folder(), "empty").unwrap()) } pub fn open_multisig(&mut self) -> &mut MultisigDatabase { - self.multisig.get_or_insert_with(|| { - Database::open(Self::folder(), "multisig", |key| key as &[u8]).unwrap() - }) + self.multisig + .get_or_insert_with(|| Database::open(Self::folder(), "multisig").unwrap()) } } diff --git a/parser/src/databases/txid_to_tx_data.rs b/parser/src/databases/txid_to_tx_data.rs index dc0fe8224..2ad905dfa 100644 --- a/parser/src/databases/txid_to_tx_data.rs +++ b/parser/src/databases/txid_to_tx_data.rs @@ -10,11 +10,11 @@ use rayon::prelude::*; use crate::structs::{Date, Height, TxData}; -use super::{AnyDatabaseGroup, Metadata, SizedDatabase, U8x31}; +use super::{AnyDatabaseGroup, Database as _Database, Metadata, U8x31}; type Key = U8x31; type Value = TxData; -type Database = SizedDatabase; +type Database = _Database; #[derive(Allocative)] pub struct TxidToTxData { @@ -105,9 +105,8 @@ impl TxidToTxData { pub fn open_db(&mut self, txid: &Txid) -> &mut Database { let db_index = Self::db_index(txid); - self.entry(db_index).or_insert_with(|| { - SizedDatabase::open(Self::folder(), &db_index.to_string(), |key| key).unwrap() - }) + self.entry(db_index) + .or_insert_with(|| Database::open(Self::folder(), &db_index.to_string()).unwrap()) } fn txid_to_key(txid: &Txid) -> U8x31 { diff --git a/parser/src/databases/txout_index_to_address_index.rs b/parser/src/databases/txout_index_to_address_index.rs index d5bd35911..dc582bd2b 100644 --- a/parser/src/databases/txout_index_to_address_index.rs +++ b/parser/src/databases/txout_index_to_address_index.rs @@ -9,11 +9,11 @@ use rayon::prelude::*; use crate::structs::{Date, Height, TxoutIndex}; -use super::{AnyDatabaseGroup, Metadata, SizedDatabase}; +use super::{AnyDatabaseGroup, Database as _Database, Metadata}; type Key = TxoutIndex; type Value = u32; -type Database = SizedDatabase; +type Database = _Database; #[derive(Allocative)] pub struct TxoutIndexToAddressIndex { @@ -77,7 +77,7 @@ impl TxoutIndexToAddressIndex { (db_index + 1) * DB_MAX_SIZE ); - SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap() + Database::open(Self::folder(), &db_name).unwrap() }) } diff --git a/parser/src/databases/txout_index_to_amount.rs b/parser/src/databases/txout_index_to_amount.rs index 473045ad8..432385916 100644 --- a/parser/src/databases/txout_index_to_amount.rs +++ b/parser/src/databases/txout_index_to_amount.rs @@ -9,11 +9,11 @@ use rayon::prelude::*; use crate::structs::{Amount, Date, Height, TxoutIndex}; -use super::{AnyDatabaseGroup, Metadata, SizedDatabase}; +use super::{AnyDatabaseGroup, Database as _Database, Metadata}; type Key = TxoutIndex; type Value = Amount; -type Database = SizedDatabase; +type Database = _Database; #[derive(Allocative)] pub struct TxoutIndexToAmount { @@ -77,7 +77,7 @@ impl TxoutIndexToAmount { (db_index + 1) * DB_MAX_SIZE ); - SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap() + Database::open(Self::folder(), &db_name).unwrap() }) } diff --git a/parser/src/io/serialization.rs b/parser/src/io/serialization.rs index 8b2e29bb2..3a4d48688 100644 --- a/parser/src/io/serialization.rs +++ b/parser/src/io/serialization.rs @@ -2,6 +2,7 @@ use std::{fmt::Debug, fs, path::Path}; use allocative::Allocative; use bincode::{Decode, Encode}; +use color_eyre::eyre::eyre; use serde::{de::DeserializeOwned, Serialize}; use crate::io::{Binary, Json}; @@ -61,7 +62,7 @@ impl Serialization { return Binary::import(compressed_bin_path); } - panic!("Wrong path") + Err(eyre!("Wrong path or no file")) } } Serialization::Json => { @@ -76,7 +77,7 @@ impl Serialization { return Json::import(json_path); } - panic!("Wrong path") + Err(eyre!("Wrong path or no file")) } } } diff --git a/parser/src/states/counters.rs b/parser/src/states/counters.rs index 4e369e376..1a20c3d3c 100644 --- a/parser/src/states/counters.rs +++ b/parser/src/states/counters.rs @@ -8,6 +8,7 @@ use super::AnyState; #[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)] pub struct Counters { + pub multisig_addresses: Counter, pub op_return_addresses: Counter, pub push_only_addresses: Counter, pub unknown_addresses: Counter, @@ -22,7 +23,7 @@ impl AnyState for Counters { } fn clear(&mut self) { - self.op_return_addresses.reset(); + self.multisig_addresses.reset(); self.push_only_addresses.reset(); self.unknown_addresses.reset(); self.empty_addresses.reset(); diff --git a/parser/src/structs/address.rs b/parser/src/structs/address.rs index 2d18f207c..12320dda5 100644 --- a/parser/src/structs/address.rs +++ b/parser/src/structs/address.rs @@ -1,11 +1,7 @@ use bitcoin_hashes::{hash160, Hash}; use biter::bitcoin::TxOut; -use itertools::Itertools; -use crate::{ - databases::{U8x19, U8x31, SANAKIRJA_MAX_KEY_SIZE}, - utils::multisig_addresses, -}; +use crate::databases::{U8x19, U8x31}; use super::{AddressType, Counter}; @@ -17,7 +13,7 @@ pub enum Address { PushOnly(u32), Unknown(u32), // https://mempool.space/tx/274f8be3b7b9b1a220285f5f71f61e2691dd04df9d69bb02a8b3b85f91fb1857 - MultiSig(Box<[u8]>), + MultiSig(u32), P2PK((u16, U8x19)), P2PKH((u16, U8x19)), P2SH((u16, U8x19)), @@ -45,6 +41,7 @@ impl Address { pub fn from( txout: &TxOut, + multisig_addresses: &mut Counter, op_return_addresses: &mut Counter, push_only_addresses: &mut Counter, unknown_addresses: &mut Counter, @@ -92,22 +89,11 @@ impl Address { Self::OpReturn(index) } else if script.is_multisig() { - let vec = multisig_addresses(script); + let index = multisig_addresses.inner(); - if vec.is_empty() { - dbg!(txout); - panic!("Multisig addresses cannot be empty !"); - } + multisig_addresses.increment(); - let mut vec = vec.into_iter().sorted_unstable().concat(); - - // TODO: Terrible! Store everything instead of only the 510 first bytes but how - // Sanakirja key limit is [u8; 510] and some multisig transactions have 999 keys - if vec.len() > SANAKIRJA_MAX_KEY_SIZE { - vec = vec.drain(..SANAKIRJA_MAX_KEY_SIZE).collect_vec(); - } - - Self::MultiSig(vec.into()) + Self::MultiSig(index) } else if script.is_push_only() { let index = push_only_addresses.inner(); diff --git a/parser/src/utils/mod.rs b/parser/src/utils/mod.rs index eb0bcbd87..18c6f8d80 100644 --- a/parser/src/utils/mod.rs +++ b/parser/src/utils/mod.rs @@ -2,7 +2,6 @@ mod consts; mod flamegraph; mod log; mod lossy; -mod multisig; mod percentile; mod retry; mod rpc; @@ -12,7 +11,6 @@ pub use consts::*; pub use flamegraph::*; pub use log::*; pub use lossy::*; -pub use multisig::*; pub use percentile::*; pub use retry::*; pub use rpc::*; diff --git a/parser/src/utils/multisig.rs b/parser/src/utils/multisig.rs deleted file mode 100644 index 22f8520de..000000000 --- a/parser/src/utils/multisig.rs +++ /dev/null @@ -1,57 +0,0 @@ -// -// Code from bitcoin-explorer now deprecated -// - -use biter::bitcoin::{ - blockdata::{ - opcodes::all, - script::Instruction::{self, Op, PushBytes}, - }, - Opcode, Script, -}; - -/// -/// Obtain addresses for multisig transactions. -/// -pub fn multisig_addresses(script: &Script) -> Vec> { - let ops: Vec = script.instructions().filter_map(|o| o.ok()).collect(); - - // obtain number of keys - let num_keys = { - if let Some(Op(op)) = ops.get(ops.len() - 2) { - decode_from_op_n(op) - } else { - unreachable!() - } - }; - - // read public keys - let mut public_keys = Vec::with_capacity(num_keys as usize); - - for op in ops.iter().skip(1).take(num_keys as usize) { - if let PushBytes(data) = op { - public_keys.push(data.as_bytes().to_vec()); - } else { - unreachable!() - } - } - - public_keys -} - -/// -/// Decode OP_N -/// -/// translated from BitcoinJ: -/// [decodeFromOpN()](https://github.com/bitcoinj/bitcoinj/blob/d3d5edbcbdb91b25de4df3b6ed6740d7e2329efc/core/src/main/java/org/bitcoinj/script/Script.java#L515:L524) -/// -#[inline] -fn decode_from_op_n(op: &Opcode) -> i32 { - if op.eq(&all::OP_PUSHBYTES_0) { - 0 - } else if op.eq(&all::OP_PUSHNUM_NEG1) { - -1 - } else { - op.to_u8() as i32 + 1 - all::OP_PUSHNUM_1.to_u8() as i32 - } -} diff --git a/server/Cargo.lock b/server/Cargo.lock index d2b6342af..fb81ed3bc 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -276,6 +276,15 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bincode" version = "2.0.0-rc.3" @@ -395,9 +404,12 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "brotli" @@ -446,9 +458,9 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.90" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" dependencies = [ "jobserver", "libc", @@ -675,6 +687,15 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + [[package]] name = "either" version = "1.12.0" @@ -889,6 +910,44 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "heed" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc30da4a93ff8cb98e535d595d6de42731d4719d707bc1c86f579158751a24e" +dependencies = [ + "bitflags 2.6.0", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" +dependencies = [ + "bincode 1.3.3", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1151,9 +1210,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -1187,9 +1246,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "linux-raw-sys" @@ -1197,6 +1256,17 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lmdb-master-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57640c190703d5ccf4a86aff4aeb749b2d287a8cb1723c76b51f39d77ab53b24" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + [[package]] name = "lock_api" version = "0.4.11" @@ -1340,7 +1410,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -1393,6 +1463,16 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1446,13 +1526,14 @@ name = "parser" version = "0.3.0" dependencies = [ "allocative", - "bincode", + "bincode 2.0.0-rc.3", "bitcoin_hashes", "biter", "chrono", "clap", "color-eyre", "derive_deref", + "heed", "inferno", "itertools", "memory-stats", @@ -1472,6 +1553,48 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -1723,7 +1846,7 @@ version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -1930,7 +2053,7 @@ name = "server" version = "0.3.0" dependencies = [ "axum", - "bincode", + "bincode 2.0.0-rc.3", "color-eyre", "derive_deref", "itertools", @@ -1961,6 +2084,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -2044,6 +2173,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -2243,7 +2381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "async-compression", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes", "futures-core", "http", @@ -2351,9 +2489,9 @@ checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna",