parser: rm multisig db

This commit is contained in:
k
2024-08-05 11:47:52 +02:00
parent 9a8f5edd58
commit ce1fed8c16
16 changed files with 369 additions and 205 deletions
+143 -6
View File
@@ -170,6 +170,15 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bincode"
version = "2.0.0-rc.3"
@@ -292,6 +301,9 @@ name = "bitflags"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
dependencies = [
"serde",
]
[[package]]
name = "bumpalo"
@@ -319,13 +331,12 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
[[package]]
name = "cc"
version = "1.0.99"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695"
checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc"
dependencies = [
"jobserver",
"libc",
"once_cell",
]
[[package]]
@@ -540,6 +551,15 @@ dependencies = [
"syn 2.0.72",
]
[[package]]
name = "doxygen-rs"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
dependencies = [
"phf",
]
[[package]]
name = "either"
version = "1.12.0"
@@ -744,6 +764,44 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "heed"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bc30da4a93ff8cb98e535d595d6de42731d4719d707bc1c86f579158751a24e"
dependencies = [
"bitflags 2.6.0",
"byteorder",
"heed-traits",
"heed-types",
"libc",
"lmdb-master-sys",
"once_cell",
"page_size",
"serde",
"synchronoise",
"url",
]
[[package]]
name = "heed-traits"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
[[package]]
name = "heed-types"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114"
dependencies = [
"bincode 1.3.3",
"byteorder",
"heed-traits",
"serde",
"serde_json",
]
[[package]]
name = "hermit-abi"
version = "0.3.9"
@@ -1045,6 +1103,17 @@ version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
[[package]]
name = "lmdb-master-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57640c190703d5ccf4a86aff4aeb749b2d287a8cb1723c76b51f39d77ab53b24"
dependencies = [
"cc",
"doxygen-rs",
"libc",
]
[[package]]
name = "lock_api"
version = "0.4.12"
@@ -1235,6 +1304,16 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "page_size"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
@@ -1278,13 +1357,14 @@ name = "parser"
version = "0.3.0"
dependencies = [
"allocative",
"bincode",
"bincode 2.0.0-rc.3",
"bitcoin_hashes",
"biter",
"chrono",
"clap",
"color-eyre",
"derive_deref",
"heed",
"inferno",
"itertools",
"memory-stats",
@@ -1304,6 +1384,48 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "phf"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
dependencies = [
"phf_macros",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
dependencies = [
"phf_shared",
"rand",
]
[[package]]
name = "phf_macros"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.72",
]
[[package]]
name = "phf_shared"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project"
version = "1.1.5"
@@ -1721,6 +1843,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "siphasher"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
[[package]]
name = "slab"
version = "0.4.9"
@@ -1798,6 +1926,15 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
[[package]]
name = "synchronoise"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"
dependencies = [
"crossbeam-queue",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
@@ -2071,9 +2208,9 @@ checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129"
[[package]]
name = "url"
version = "2.5.0"
version = "2.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
dependencies = [
"form_urlencoded",
"idna",
+1
View File
@@ -28,3 +28,4 @@ serde = { version = "1.0.204", features = ["derive"] }
serde_json = "1.0.122"
toml = "0.8.19"
zstd = "0.13.2"
heed = "0.20.3"
+7 -1
View File
@@ -129,6 +129,7 @@ pub fn parse(
let mut txouts_parsing_results = pre_process_outputs(
&block,
compute_addresses,
&mut states.address_counters.multisig_addresses,
&mut states.address_counters.op_return_addresses,
&mut states.address_counters.push_only_addresses,
&mut states.address_counters.unknown_addresses,
@@ -341,8 +342,10 @@ pub fn parse(
if input_tx_data.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| rpc
.get_tx_out(&input_txid, input_vout, None)
.get_raw_transaction(&input_txid, None)
.unwrap()
.output
.get(input_vout as usize)
.unwrap()
.value
.to_sat()
@@ -774,9 +777,11 @@ pub struct TxoutsParsingResults {
op_returns: usize,
}
#[allow(clippy::too_many_arguments)]
fn pre_process_outputs(
block: &Block,
compute_addresses: bool,
multisig_addresses: &mut Counter,
op_return_addresses: &mut Counter,
push_only_addresses: &mut Counter,
unknown_addresses: &mut Counter,
@@ -821,6 +826,7 @@ fn pre_process_outputs(
let address_opt = compute_addresses.then(|| {
let address = Address::from(
txout,
multisig_addresses,
op_return_addresses,
push_only_addresses,
unknown_addresses,
+26 -55
View File
@@ -15,53 +15,37 @@ use derive_deref::{Deref, DerefMut};
//
// Possible compression: https://pijul.org/posts/sanakirja-zstd/
use sanakirja::{
btree::{self, page, page_unsized, BTreeMutPage, Db_},
btree::{self, page, Db_},
direct_repr, Commit, Env, Error, MutTxn, RootDb, Storable, UnsizedStorable,
};
use crate::io::OUTPUTS_FOLDER_PATH;
pub type SizedDatabase<Key, Value> = Database<Key, Key, Value, page::Page<Key, Value>>;
pub type UnsizedDatabase<KeyTree, KeyDB, Value> =
Database<KeyTree, KeyDB, Value, page_unsized::Page<KeyDB, Value>>;
#[derive(Allocative)]
#[allocative(bound = "KeyTree: Allocative, KeyDB, Value: Allocative, Page")]
#[allocative(bound = "Key: Allocative, Value: Allocative")]
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way.
pub struct Database<KeyTree, KeyDB, Value, Page>
pub struct Database<Key, Value>
where
KeyTree: Ord + Clone + Debug,
KeyDB: Ord + ?Sized + Storable,
Key: Ord + Clone + Debug + ?Sized + Storable,
Value: Storable + PartialEq,
Page: BTreeMutPage<KeyDB, Value>,
{
pub cached_puts: BTreeMap<KeyTree, Value>,
pub cached_dels: BTreeSet<KeyTree>,
pub cached_puts: BTreeMap<Key, Value>,
pub cached_dels: BTreeSet<Key>,
#[allocative(skip)]
db: Db_<KeyDB, Value, Page>,
db: Db_<Key, Value, page::Page<Key, Value>>,
#[allocative(skip)]
txn: MutTxn<Env, ()>,
#[allocative(skip)]
key_tree_to_key_db: fn(&KeyTree) -> &KeyDB,
}
pub const SANAKIRJA_MAX_KEY_SIZE: usize = 510;
const ROOT_DB: usize = 0;
const PAGE_SIZE: u64 = 4096 * 256; // 1mo - Must be a multiplier of 4096
impl<KeyDB, KeyTree, Value, Page> Database<KeyTree, KeyDB, Value, Page>
impl<Key, Value> Database<Key, Value>
where
KeyTree: Ord + Clone + Debug,
KeyDB: Ord + ?Sized + Storable,
Key: Ord + Clone + Debug + ?Sized + Storable,
Value: Storable + PartialEq,
Page: BTreeMutPage<KeyDB, Value>,
{
pub fn open(
folder: &str,
file: &str,
key_tree_to_key_db: fn(&KeyTree) -> &KeyDB,
) -> color_eyre::Result<Self> {
pub fn open(folder: &str, file: &str) -> color_eyre::Result<Self> {
let mut txn = Self::init_txn(folder, file)?;
let db = txn
@@ -73,20 +57,19 @@ where
cached_dels: BTreeSet::default(),
db,
txn,
key_tree_to_key_db,
})
}
pub fn iter<F>(&self, callback: &mut F)
where
F: FnMut((&KeyDB, &Value)),
F: FnMut((&Key, &Value)),
{
btree::iter(&self.txn, &self.db, None)
.unwrap()
.for_each(|entry| callback(entry.unwrap()));
}
pub fn get(&self, key: &KeyTree) -> Option<&Value> {
pub fn get(&self, key: &Key) -> Option<&Value> {
if let Some(cached_put) = self.get_from_puts(key) {
return Some(cached_put);
}
@@ -94,13 +77,11 @@ where
self.db_get(key)
}
pub fn db_get(&self, key: &KeyTree) -> Option<&Value> {
let k = (self.key_tree_to_key_db)(key);
pub fn db_get(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
let option = btree::get(&self.txn, &self.db, k, None).unwrap();
if let Some((k_found, v)) = option {
if k == k_found {
if let Some((key_found, v)) = option {
if key == key_found {
return Some(v);
}
}
@@ -109,17 +90,17 @@ where
}
#[inline(always)]
pub fn get_from_puts(&self, key: &KeyTree) -> Option<&Value> {
pub fn get_from_puts(&self, key: &Key) -> Option<&Value> {
self.cached_puts.get(key)
}
#[inline(always)]
pub fn get_mut_from_puts(&mut self, key: &KeyTree) -> Option<&mut Value> {
pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> {
self.cached_puts.get_mut(key)
}
#[inline(always)]
pub fn remove(&mut self, key: &KeyTree) -> Option<Value> {
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.remove_from_puts(key).or_else(|| {
self.db_remove(key);
@@ -128,30 +109,30 @@ where
}
#[inline(always)]
pub fn db_remove(&mut self, key: &KeyTree) {
pub fn db_remove(&mut self, key: &Key) {
self.cached_dels.insert(key.clone());
}
pub fn update(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.insert(key.clone());
self.cached_puts.insert(key, value)
}
#[inline(always)]
pub fn remove_from_puts(&mut self, key: &KeyTree) -> Option<Value> {
pub fn remove_from_puts(&mut self, key: &Key) -> Option<Value> {
self.cached_puts.remove(key)
}
#[inline(always)]
pub fn insert(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.remove(&key);
self.unsafe_insert(key, value)
}
#[inline(always)]
pub fn unsafe_insert(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_puts.insert(key, value)
}
@@ -175,12 +156,7 @@ where
self.cached_dels
.into_iter()
.try_for_each(|key| -> Result<(), Error> {
btree::del(
&mut self.txn,
&mut self.db,
(self.key_tree_to_key_db)(&key),
None,
)?;
btree::del(&mut self.txn, &mut self.db, &key, None)?;
Ok(())
})?;
@@ -188,12 +164,7 @@ where
self.cached_puts
.into_iter()
.try_for_each(|(key, value)| -> Result<(), Error> {
btree::put(
&mut self.txn,
&mut self.db,
(self.key_tree_to_key_db)(&key),
&value,
)?;
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
})?;
@@ -12,11 +12,11 @@ use crate::{
utils::time,
};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = u32;
type Value = AddressData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct AddressIndexToAddressData {
@@ -78,7 +78,7 @@ impl AddressIndexToAddressData {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}
@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Date, EmptyAddressData, Height};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = u32;
type Value = EmptyAddressData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct AddressIndexToEmptyAddressData {
@@ -45,12 +45,6 @@ impl AddressIndexToEmptyAddressData {
self.open_db(&key).unsafe_insert(key, value)
}
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
// self.metadata.called_remove();
// self.open_db(key).remove_from_puts(key)
// }
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.metadata.called_remove();
@@ -86,7 +80,7 @@ impl AddressIndexToEmptyAddressData {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}
@@ -5,16 +5,12 @@ use rayon::prelude::*;
use crate::structs::{Address, Date, Height};
use super::{
AnyDatabaseGroup, Database, Metadata, SizedDatabase, U8x19, U8x31,
UnsizedDatabase as _UnsizedDatabase,
};
use super::{AnyDatabaseGroup, Database, Metadata, U8x19, U8x31};
type Value = u32;
type U8x19Database = SizedDatabase<U8x19, Value>;
type U8x31Database = SizedDatabase<U8x31, Value>;
type U32Database = SizedDatabase<u32, Value>;
type UnsizedDatabase = _UnsizedDatabase<Box<[u8]>, [u8], Value>;
type U8x19Database = Database<U8x19, Value>;
type U8x31Database = Database<U8x31, Value>;
type U32Database = Database<u32, Value>;
type P2PKDatabase = U8x19Database;
type P2PKHDatabase = U8x19Database;
@@ -26,7 +22,7 @@ type UnknownDatabase = U32Database;
type OpReturnDatabase = U32Database;
type PushOnlyDatabase = U32Database;
type EmptyDatabase = U32Database;
type MultisigDatabase = UnsizedDatabase;
type MultisigDatabase = U32Database;
#[derive(Allocative)]
pub struct AddressToAddressIndex {
@@ -106,7 +102,7 @@ impl AddressToAddressIndex {
Address::Unknown(key) => self.unknown.as_ref().unwrap().get(key),
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get(key),
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get(key),
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get(key),
Address::MultiSig(key) => self.push_only.as_ref().unwrap().get(key),
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get(key),
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get(key),
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get(key),
@@ -155,7 +151,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2pk"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -166,7 +161,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2pkh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -177,7 +171,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2sh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -188,7 +181,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2wpkh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -199,7 +191,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2wsh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -210,7 +201,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2tr"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -218,28 +208,27 @@ impl AddressToAddressIndex {
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
self.unknown
.get_or_insert_with(|| Database::open(Self::folder(), "unknown", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "unknown").unwrap())
}
pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
self.op_return
.get_or_insert_with(|| Database::open(Self::folder(), "op_return", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "op_return").unwrap())
}
pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
self.push_only
.get_or_insert_with(|| Database::open(Self::folder(), "push_only", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "push_only").unwrap())
}
pub fn open_empty(&mut self) -> &mut UnknownDatabase {
self.empty
.get_or_insert_with(|| Database::open(Self::folder(), "empty", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "empty").unwrap())
}
pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
self.multisig.get_or_insert_with(|| {
Database::open(Self::folder(), "multisig", |key| key as &[u8]).unwrap()
})
self.multisig
.get_or_insert_with(|| Database::open(Self::folder(), "multisig").unwrap())
}
}
+4 -5
View File
@@ -10,11 +10,11 @@ use rayon::prelude::*;
use crate::structs::{Date, Height, TxData};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase, U8x31};
use super::{AnyDatabaseGroup, Database as _Database, Metadata, U8x31};
type Key = U8x31;
type Value = TxData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxidToTxData {
@@ -105,9 +105,8 @@ impl TxidToTxData {
pub fn open_db(&mut self, txid: &Txid) -> &mut Database {
let db_index = Self::db_index(txid);
self.entry(db_index).or_insert_with(|| {
SizedDatabase::open(Self::folder(), &db_index.to_string(), |key| key).unwrap()
})
self.entry(db_index)
.or_insert_with(|| Database::open(Self::folder(), &db_index.to_string()).unwrap())
}
fn txid_to_key(txid: &Txid) -> U8x31 {
@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = u32;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxoutIndexToAddressIndex {
@@ -77,7 +77,7 @@ impl TxoutIndexToAddressIndex {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}
@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Amount, Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = Amount;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxoutIndexToAmount {
@@ -77,7 +77,7 @@ impl TxoutIndexToAmount {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}
+3 -2
View File
@@ -2,6 +2,7 @@ use std::{fmt::Debug, fs, path::Path};
use allocative::Allocative;
use bincode::{Decode, Encode};
use color_eyre::eyre::eyre;
use serde::{de::DeserializeOwned, Serialize};
use crate::io::{Binary, Json};
@@ -61,7 +62,7 @@ impl Serialization {
return Binary::import(compressed_bin_path);
}
panic!("Wrong path")
Err(eyre!("Wrong path or no file"))
}
}
Serialization::Json => {
@@ -76,7 +77,7 @@ impl Serialization {
return Json::import(json_path);
}
panic!("Wrong path")
Err(eyre!("Wrong path or no file"))
}
}
}
+2 -1
View File
@@ -8,6 +8,7 @@ use super::AnyState;
#[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)]
pub struct Counters {
pub multisig_addresses: Counter,
pub op_return_addresses: Counter,
pub push_only_addresses: Counter,
pub unknown_addresses: Counter,
@@ -22,7 +23,7 @@ impl AnyState for Counters {
}
fn clear(&mut self) {
self.op_return_addresses.reset();
self.multisig_addresses.reset();
self.push_only_addresses.reset();
self.unknown_addresses.reset();
self.empty_addresses.reset();
+6 -20
View File
@@ -1,11 +1,7 @@
use bitcoin_hashes::{hash160, Hash};
use biter::bitcoin::TxOut;
use itertools::Itertools;
use crate::{
databases::{U8x19, U8x31, SANAKIRJA_MAX_KEY_SIZE},
utils::multisig_addresses,
};
use crate::databases::{U8x19, U8x31};
use super::{AddressType, Counter};
@@ -17,7 +13,7 @@ pub enum Address {
PushOnly(u32),
Unknown(u32),
// https://mempool.space/tx/274f8be3b7b9b1a220285f5f71f61e2691dd04df9d69bb02a8b3b85f91fb1857
MultiSig(Box<[u8]>),
MultiSig(u32),
P2PK((u16, U8x19)),
P2PKH((u16, U8x19)),
P2SH((u16, U8x19)),
@@ -45,6 +41,7 @@ impl Address {
pub fn from(
txout: &TxOut,
multisig_addresses: &mut Counter,
op_return_addresses: &mut Counter,
push_only_addresses: &mut Counter,
unknown_addresses: &mut Counter,
@@ -92,22 +89,11 @@ impl Address {
Self::OpReturn(index)
} else if script.is_multisig() {
let vec = multisig_addresses(script);
let index = multisig_addresses.inner();
if vec.is_empty() {
dbg!(txout);
panic!("Multisig addresses cannot be empty !");
}
multisig_addresses.increment();
let mut vec = vec.into_iter().sorted_unstable().concat();
// TODO: Terrible! Store everything instead of only the 510 first bytes but how
// Sanakirja key limit is [u8; 510] and some multisig transactions have 999 keys
if vec.len() > SANAKIRJA_MAX_KEY_SIZE {
vec = vec.drain(..SANAKIRJA_MAX_KEY_SIZE).collect_vec();
}
Self::MultiSig(vec.into())
Self::MultiSig(index)
} else if script.is_push_only() {
let index = push_only_addresses.inner();
-2
View File
@@ -2,7 +2,6 @@ mod consts;
mod flamegraph;
mod log;
mod lossy;
mod multisig;
mod percentile;
mod retry;
mod rpc;
@@ -12,7 +11,6 @@ pub use consts::*;
pub use flamegraph::*;
pub use log::*;
pub use lossy::*;
pub use multisig::*;
pub use percentile::*;
pub use retry::*;
pub use rpc::*;
-57
View File
@@ -1,57 +0,0 @@
//
// Code from bitcoin-explorer now deprecated
//
use biter::bitcoin::{
blockdata::{
opcodes::all,
script::Instruction::{self, Op, PushBytes},
},
Opcode, Script,
};
///
/// Obtain addresses for multisig transactions.
///
pub fn multisig_addresses(script: &Script) -> Vec<Vec<u8>> {
let ops: Vec<Instruction> = script.instructions().filter_map(|o| o.ok()).collect();
// obtain number of keys
let num_keys = {
if let Some(Op(op)) = ops.get(ops.len() - 2) {
decode_from_op_n(op)
} else {
unreachable!()
}
};
// read public keys
let mut public_keys = Vec::with_capacity(num_keys as usize);
for op in ops.iter().skip(1).take(num_keys as usize) {
if let PushBytes(data) = op {
public_keys.push(data.as_bytes().to_vec());
} else {
unreachable!()
}
}
public_keys
}
///
/// Decode OP_N
///
/// translated from BitcoinJ:
/// [decodeFromOpN()](https://github.com/bitcoinj/bitcoinj/blob/d3d5edbcbdb91b25de4df3b6ed6740d7e2329efc/core/src/main/java/org/bitcoinj/script/Script.java#L515:L524)
///
#[inline]
fn decode_from_op_n(op: &Opcode) -> i32 {
if op.eq(&all::OP_PUSHBYTES_0) {
0
} else if op.eq(&all::OP_PUSHNUM_NEG1) {
-1
} else {
op.to_u8() as i32 + 1 - all::OP_PUSHNUM_1.to_u8() as i32
}
}
+153 -15
View File
@@ -276,6 +276,15 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bincode"
version = "2.0.0-rc.3"
@@ -395,9 +404,12 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.5.0"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
dependencies = [
"serde",
]
[[package]]
name = "brotli"
@@ -446,9 +458,9 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
[[package]]
name = "cc"
version = "1.0.90"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5"
checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc"
dependencies = [
"jobserver",
"libc",
@@ -675,6 +687,15 @@ dependencies = [
"syn 2.0.60",
]
[[package]]
name = "doxygen-rs"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
dependencies = [
"phf",
]
[[package]]
name = "either"
version = "1.12.0"
@@ -889,6 +910,44 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "heed"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bc30da4a93ff8cb98e535d595d6de42731d4719d707bc1c86f579158751a24e"
dependencies = [
"bitflags 2.6.0",
"byteorder",
"heed-traits",
"heed-types",
"libc",
"lmdb-master-sys",
"once_cell",
"page_size",
"serde",
"synchronoise",
"url",
]
[[package]]
name = "heed-traits"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
[[package]]
name = "heed-types"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114"
dependencies = [
"bincode 1.3.3",
"byteorder",
"heed-traits",
"serde",
"serde_json",
]
[[package]]
name = "hermit-abi"
version = "0.3.9"
@@ -1151,9 +1210,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
[[package]]
name = "jobserver"
version = "0.1.28"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6"
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
dependencies = [
"libc",
]
@@ -1187,9 +1246,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.153"
version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
[[package]]
name = "linux-raw-sys"
@@ -1197,6 +1256,17 @@ version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
[[package]]
name = "lmdb-master-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57640c190703d5ccf4a86aff4aeb749b2d287a8cb1723c76b51f39d77ab53b24"
dependencies = [
"cc",
"doxygen-rs",
"libc",
]
[[package]]
name = "lock_api"
version = "0.4.11"
@@ -1340,7 +1410,7 @@ version = "0.10.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
dependencies = [
"bitflags 2.5.0",
"bitflags 2.6.0",
"cfg-if",
"foreign-types",
"libc",
@@ -1393,6 +1463,16 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "page_size"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
@@ -1446,13 +1526,14 @@ name = "parser"
version = "0.3.0"
dependencies = [
"allocative",
"bincode",
"bincode 2.0.0-rc.3",
"bitcoin_hashes",
"biter",
"chrono",
"clap",
"color-eyre",
"derive_deref",
"heed",
"inferno",
"itertools",
"memory-stats",
@@ -1472,6 +1553,48 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "phf"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
dependencies = [
"phf_macros",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
dependencies = [
"phf_shared",
"rand",
]
[[package]]
name = "phf_macros"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.60",
]
[[package]]
name = "phf_shared"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project"
version = "1.1.5"
@@ -1723,7 +1846,7 @@ version = "0.38.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89"
dependencies = [
"bitflags 2.5.0",
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys",
@@ -1930,7 +2053,7 @@ name = "server"
version = "0.3.0"
dependencies = [
"axum",
"bincode",
"bincode 2.0.0-rc.3",
"color-eyre",
"derive_deref",
"itertools",
@@ -1961,6 +2084,12 @@ dependencies = [
"libc",
]
[[package]]
name = "siphasher"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
[[package]]
name = "slab"
version = "0.4.9"
@@ -2044,6 +2173,15 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
[[package]]
name = "synchronoise"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"
dependencies = [
"crossbeam-queue",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
@@ -2243,7 +2381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
dependencies = [
"async-compression",
"bitflags 2.5.0",
"bitflags 2.6.0",
"bytes",
"futures-core",
"http",
@@ -2351,9 +2489,9 @@ checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129"
[[package]]
name = "url"
version = "2.5.0"
version = "2.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
dependencies = [
"form_urlencoded",
"idna",