Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-30 09:30:00 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: rm multisig db

Browse Source
This commit is contained in:
k
2024-08-05 11:47:52 +02:00
parent 9a8f5edd58
commit ce1fed8c16
16 changed files with 369 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
parser/src/actions/parse.rs
View File

@@ -129,6 +129,7 @@ pub fn parse(
let mut txouts_parsing_results = pre_process_outputs(
&block,
compute_addresses,
&mut states.address_counters.multisig_addresses,
&mut states.address_counters.op_return_addresses,
&mut states.address_counters.push_only_addresses,
&mut states.address_counters.unknown_addresses,
@@ -341,8 +342,10 @@ pub fn parse(
if input_tx_data.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| rpc
.get_tx_out(&input_txid, input_vout, None)
.get_raw_transaction(&input_txid, None)
.unwrap()
.output
.get(input_vout as usize)
.unwrap()
.value
.to_sat()
@@ -774,9 +777,11 @@ pub struct TxoutsParsingResults {
op_returns: usize,
}
#[allow(clippy::too_many_arguments)]
fn pre_process_outputs(
block: &Block,
compute_addresses: bool,
multisig_addresses: &mut Counter,
op_return_addresses: &mut Counter,
push_only_addresses: &mut Counter,
unknown_addresses: &mut Counter,
@@ -821,6 +826,7 @@ fn pre_process_outputs(
let address_opt = compute_addresses.then(|| {
let address = Address::from(
txout,
multisig_addresses,
op_return_addresses,
push_only_addresses,
unknown_addresses,

81
parser/src/databases/_database.rs
View File

@@ -15,53 +15,37 @@ use derive_deref::{Deref, DerefMut};
//
// Possible compression: https://pijul.org/posts/sanakirja-zstd/
use sanakirja::{
btree::{self, page, page_unsized, BTreeMutPage, Db_},
btree::{self, page, Db_},
direct_repr, Commit, Env, Error, MutTxn, RootDb, Storable, UnsizedStorable,
};
use crate::io::OUTPUTS_FOLDER_PATH;
pub type SizedDatabase<Key, Value> = Database<Key, Key, Value, page::Page<Key, Value>>;
pub type UnsizedDatabase<KeyTree, KeyDB, Value> =
Database<KeyTree, KeyDB, Value, page_unsized::Page<KeyDB, Value>>;
#[derive(Allocative)]
#[allocative(bound = "KeyTree: Allocative, KeyDB, Value: Allocative, Page")]
#[allocative(bound = "Key: Allocative, Value: Allocative")]
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way.
pub struct Database<KeyTree, KeyDB, Value, Page>
pub struct Database<Key, Value>
where
KeyTree: Ord + Clone + Debug,
KeyDB: Ord + ?Sized + Storable,
Key: Ord + Clone + Debug + ?Sized + Storable,
Value: Storable + PartialEq,
Page: BTreeMutPage<KeyDB, Value>,
{
pub cached_puts: BTreeMap<KeyTree, Value>,
pub cached_dels: BTreeSet<KeyTree>,
pub cached_puts: BTreeMap<Key, Value>,
pub cached_dels: BTreeSet<Key>,
#[allocative(skip)]
db: Db_<KeyDB, Value, Page>,
db: Db_<Key, Value, page::Page<Key, Value>>,
#[allocative(skip)]
txn: MutTxn<Env, ()>,
#[allocative(skip)]
key_tree_to_key_db: fn(&KeyTree) -> &KeyDB,
}
pub const SANAKIRJA_MAX_KEY_SIZE: usize = 510;
const ROOT_DB: usize = 0;
const PAGE_SIZE: u64 = 4096 * 256; // 1mo - Must be a multiplier of 4096
impl<KeyDB, KeyTree, Value, Page> Database<KeyTree, KeyDB, Value, Page>
impl<Key, Value> Database<Key, Value>
where
KeyTree: Ord + Clone + Debug,
KeyDB: Ord + ?Sized + Storable,
Key: Ord + Clone + Debug + ?Sized + Storable,
Value: Storable + PartialEq,
Page: BTreeMutPage<KeyDB, Value>,
{
pub fn open(
folder: &str,
file: &str,
key_tree_to_key_db: fn(&KeyTree) -> &KeyDB,
) -> color_eyre::Result<Self> {
pub fn open(folder: &str, file: &str) -> color_eyre::Result<Self> {
let mut txn = Self::init_txn(folder, file)?;
let db = txn
@@ -73,20 +57,19 @@ where
cached_dels: BTreeSet::default(),
db,
txn,
key_tree_to_key_db,
})
}
pub fn iter<F>(&self, callback: &mut F)
where
F: FnMut((&KeyDB, &Value)),
F: FnMut((&Key, &Value)),
{
btree::iter(&self.txn, &self.db, None)
.unwrap()
.for_each(|entry| callback(entry.unwrap()));
}
pub fn get(&self, key: &KeyTree) -> Option<&Value> {
pub fn get(&self, key: &Key) -> Option<&Value> {
if let Some(cached_put) = self.get_from_puts(key) {
return Some(cached_put);
}
@@ -94,13 +77,11 @@ where
self.db_get(key)
}
pub fn db_get(&self, key: &KeyTree) -> Option<&Value> {
let k = (self.key_tree_to_key_db)(key);
pub fn db_get(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
let option = btree::get(&self.txn, &self.db, k, None).unwrap();
if let Some((k_found, v)) = option {
if k == k_found {
if let Some((key_found, v)) = option {
if key == key_found {
return Some(v);
}
}
@@ -109,17 +90,17 @@ where
}
#[inline(always)]
pub fn get_from_puts(&self, key: &KeyTree) -> Option<&Value> {
pub fn get_from_puts(&self, key: &Key) -> Option<&Value> {
self.cached_puts.get(key)
}
#[inline(always)]
pub fn get_mut_from_puts(&mut self, key: &KeyTree) -> Option<&mut Value> {
pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> {
self.cached_puts.get_mut(key)
}
#[inline(always)]
pub fn remove(&mut self, key: &KeyTree) -> Option<Value> {
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.remove_from_puts(key).or_else(|| {
self.db_remove(key);
@@ -128,30 +109,30 @@ where
}
#[inline(always)]
pub fn db_remove(&mut self, key: &KeyTree) {
pub fn db_remove(&mut self, key: &Key) {
self.cached_dels.insert(key.clone());
}
pub fn update(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.insert(key.clone());
self.cached_puts.insert(key, value)
}
#[inline(always)]
pub fn remove_from_puts(&mut self, key: &KeyTree) -> Option<Value> {
pub fn remove_from_puts(&mut self, key: &Key) -> Option<Value> {
self.cached_puts.remove(key)
}
#[inline(always)]
pub fn insert(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.remove(&key);
self.unsafe_insert(key, value)
}
#[inline(always)]
pub fn unsafe_insert(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_puts.insert(key, value)
}
@@ -175,12 +156,7 @@ where
self.cached_dels
.into_iter()
.try_for_each(|key| -> Result<(), Error> {
btree::del(
&mut self.txn,
&mut self.db,
(self.key_tree_to_key_db)(&key),
None,
)?;
btree::del(&mut self.txn, &mut self.db, &key, None)?;
Ok(())
})?;
@@ -188,12 +164,7 @@ where
self.cached_puts
.into_iter()
.try_for_each(|(key, value)| -> Result<(), Error> {
btree::put(
&mut self.txn,
&mut self.db,
(self.key_tree_to_key_db)(&key),
&value,
)?;
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
})?;

6
parser/src/databases/address_index_to_address_data.rs
View File

@@ -12,11 +12,11 @@ use crate::{
utils::time,
};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = u32;
type Value = AddressData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct AddressIndexToAddressData {
@@ -78,7 +78,7 @@ impl AddressIndexToAddressData {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}

12
parser/src/databases/address_index_to_empty_address_data.rs
View File

@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Date, EmptyAddressData, Height};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = u32;
type Value = EmptyAddressData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct AddressIndexToEmptyAddressData {
@@ -45,12 +45,6 @@ impl AddressIndexToEmptyAddressData {
self.open_db(&key).unsafe_insert(key, value)
}
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
// self.metadata.called_remove();
// self.open_db(key).remove_from_puts(key)
// }
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.metadata.called_remove();
@@ -86,7 +80,7 @@ impl AddressIndexToEmptyAddressData {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}

35
parser/src/databases/address_to_address_index.rs
View File

@@ -5,16 +5,12 @@ use rayon::prelude::*;
use crate::structs::{Address, Date, Height};
use super::{
AnyDatabaseGroup, Database, Metadata, SizedDatabase, U8x19, U8x31,
UnsizedDatabase as _UnsizedDatabase,
};
use super::{AnyDatabaseGroup, Database, Metadata, U8x19, U8x31};
type Value = u32;
type U8x19Database = SizedDatabase<U8x19, Value>;
type U8x31Database = SizedDatabase<U8x31, Value>;
type U32Database = SizedDatabase<u32, Value>;
type UnsizedDatabase = _UnsizedDatabase<Box<[u8]>, [u8], Value>;
type U8x19Database = Database<U8x19, Value>;
type U8x31Database = Database<U8x31, Value>;
type U32Database = Database<u32, Value>;
type P2PKDatabase = U8x19Database;
type P2PKHDatabase = U8x19Database;
@@ -26,7 +22,7 @@ type UnknownDatabase = U32Database;
type OpReturnDatabase = U32Database;
type PushOnlyDatabase = U32Database;
type EmptyDatabase = U32Database;
type MultisigDatabase = UnsizedDatabase;
type MultisigDatabase = U32Database;
#[derive(Allocative)]
pub struct AddressToAddressIndex {
@@ -106,7 +102,7 @@ impl AddressToAddressIndex {
Address::Unknown(key) => self.unknown.as_ref().unwrap().get(key),
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get(key),
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get(key),
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get(key),
Address::MultiSig(key) => self.push_only.as_ref().unwrap().get(key),
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get(key),
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get(key),
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get(key),
@@ -155,7 +151,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2pk"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -166,7 +161,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2pkh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -177,7 +171,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2sh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -188,7 +181,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2wpkh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -199,7 +191,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2wsh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -210,7 +201,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2tr"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -218,28 +208,27 @@ impl AddressToAddressIndex {
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
self.unknown
.get_or_insert_with(|| Database::open(Self::folder(), "unknown", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "unknown").unwrap())
}
pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
self.op_return
.get_or_insert_with(|| Database::open(Self::folder(), "op_return", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "op_return").unwrap())
}
pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
self.push_only
.get_or_insert_with(|| Database::open(Self::folder(), "push_only", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "push_only").unwrap())
}
pub fn open_empty(&mut self) -> &mut UnknownDatabase {
self.empty
.get_or_insert_with(|| Database::open(Self::folder(), "empty", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "empty").unwrap())
}
pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
self.multisig.get_or_insert_with(|| {
Database::open(Self::folder(), "multisig", |key| key as &[u8]).unwrap()
})
self.multisig
.get_or_insert_with(|| Database::open(Self::folder(), "multisig").unwrap())
}
}

9
parser/src/databases/txid_to_tx_data.rs
View File

@@ -10,11 +10,11 @@ use rayon::prelude::*;
use crate::structs::{Date, Height, TxData};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase, U8x31};
use super::{AnyDatabaseGroup, Database as _Database, Metadata, U8x31};
type Key = U8x31;
type Value = TxData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxidToTxData {
@@ -105,9 +105,8 @@ impl TxidToTxData {
pub fn open_db(&mut self, txid: &Txid) -> &mut Database {
let db_index = Self::db_index(txid);
self.entry(db_index).or_insert_with(|| {
SizedDatabase::open(Self::folder(), &db_index.to_string(), |key| key).unwrap()
})
self.entry(db_index)
.or_insert_with(|| Database::open(Self::folder(), &db_index.to_string()).unwrap())
}
fn txid_to_key(txid: &Txid) -> U8x31 {

6
parser/src/databases/txout_index_to_address_index.rs
View File

@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = u32;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxoutIndexToAddressIndex {
@@ -77,7 +77,7 @@ impl TxoutIndexToAddressIndex {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}

6
parser/src/databases/txout_index_to_amount.rs
View File

@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Amount, Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = Amount;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxoutIndexToAmount {
@@ -77,7 +77,7 @@ impl TxoutIndexToAmount {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}

5
parser/src/io/serialization.rs
View File

@@ -2,6 +2,7 @@ use std::{fmt::Debug, fs, path::Path};
use allocative::Allocative;
use bincode::{Decode, Encode};
use color_eyre::eyre::eyre;
use serde::{de::DeserializeOwned, Serialize};
use crate::io::{Binary, Json};
@@ -61,7 +62,7 @@ impl Serialization {
return Binary::import(compressed_bin_path);
}
panic!("Wrong path")
Err(eyre!("Wrong path or no file"))
}
}
Serialization::Json => {
@@ -76,7 +77,7 @@ impl Serialization {
return Json::import(json_path);
}
panic!("Wrong path")
Err(eyre!("Wrong path or no file"))
}
}
}

3
parser/src/states/counters.rs
View File

@@ -8,6 +8,7 @@ use super::AnyState;
#[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)]
pub struct Counters {
pub multisig_addresses: Counter,
pub op_return_addresses: Counter,
pub push_only_addresses: Counter,
pub unknown_addresses: Counter,
@@ -22,7 +23,7 @@ impl AnyState for Counters {
}
fn clear(&mut self) {
self.op_return_addresses.reset();
self.multisig_addresses.reset();
self.push_only_addresses.reset();
self.unknown_addresses.reset();
self.empty_addresses.reset();

26
parser/src/structs/address.rs
View File

@@ -1,11 +1,7 @@
use bitcoin_hashes::{hash160, Hash};
use biter::bitcoin::TxOut;
use itertools::Itertools;
use crate::{
databases::{U8x19, U8x31, SANAKIRJA_MAX_KEY_SIZE},
utils::multisig_addresses,
};
use crate::databases::{U8x19, U8x31};
use super::{AddressType, Counter};
@@ -17,7 +13,7 @@ pub enum Address {
PushOnly(u32),
Unknown(u32),
// https://mempool.space/tx/274f8be3b7b9b1a220285f5f71f61e2691dd04df9d69bb02a8b3b85f91fb1857
MultiSig(Box<[u8]>),
MultiSig(u32),
P2PK((u16, U8x19)),
P2PKH((u16, U8x19)),
P2SH((u16, U8x19)),
@@ -45,6 +41,7 @@ impl Address {
pub fn from(
txout: &TxOut,
multisig_addresses: &mut Counter,
op_return_addresses: &mut Counter,
push_only_addresses: &mut Counter,
unknown_addresses: &mut Counter,
@@ -92,22 +89,11 @@ impl Address {
Self::OpReturn(index)
} else if script.is_multisig() {
let vec = multisig_addresses(script);
let index = multisig_addresses.inner();
if vec.is_empty() {
dbg!(txout);
panic!("Multisig addresses cannot be empty !");
}
multisig_addresses.increment();
let mut vec = vec.into_iter().sorted_unstable().concat();
// TODO: Terrible! Store everything instead of only the 510 first bytes but how
// Sanakirja key limit is [u8; 510] and some multisig transactions have 999 keys
if vec.len() > SANAKIRJA_MAX_KEY_SIZE {
vec = vec.drain(..SANAKIRJA_MAX_KEY_SIZE).collect_vec();
}
Self::MultiSig(vec.into())
Self::MultiSig(index)
} else if script.is_push_only() {
let index = push_only_addresses.inner();

2
parser/src/utils/mod.rs
View File

@@ -2,7 +2,6 @@ mod consts;
mod flamegraph;
mod log;
mod lossy;
mod multisig;
mod percentile;
mod retry;
mod rpc;
@@ -12,7 +11,6 @@ pub use consts::*;
pub use flamegraph::*;
pub use log::*;
pub use lossy::*;
pub use multisig::*;
pub use percentile::*;
pub use retry::*;
pub use rpc::*;

57
parser/src/utils/multisig.rs
View File

@@ -1,57 +0,0 @@
//
// Code from bitcoin-explorer now deprecated
//
use biter::bitcoin::{
blockdata::{
opcodes::all,
script::Instruction::{self, Op, PushBytes},
},
Opcode, Script,
};
///
/// Obtain addresses for multisig transactions.
///
pub fn multisig_addresses(script: &Script) -> Vec<Vec<u8>> {
let ops: Vec<Instruction> = script.instructions().filter_map(|o| o.ok()).collect();
// obtain number of keys
let num_keys = {
if let Some(Op(op)) = ops.get(ops.len() - 2) {
decode_from_op_n(op)
} else {
unreachable!()
}
};
// read public keys
let mut public_keys = Vec::with_capacity(num_keys as usize);
for op in ops.iter().skip(1).take(num_keys as usize) {
if let PushBytes(data) = op {
public_keys.push(data.as_bytes().to_vec());
} else {
unreachable!()
}
}
public_keys
}
///
/// Decode OP_N
///
/// translated from BitcoinJ:
/// [decodeFromOpN()](https://github.com/bitcoinj/bitcoinj/blob/d3d5edbcbdb91b25de4df3b6ed6740d7e2329efc/core/src/main/java/org/bitcoinj/script/Script.java#L515:L524)
///
#[inline]
fn decode_from_op_n(op: &Opcode) -> i32 {
if op.eq(&all::OP_PUSHBYTES_0) {
0
} else if op.eq(&all::OP_PUSHNUM_NEG1) {
-1
} else {
op.to_u8() as i32 + 1 - all::OP_PUSHNUM_1.to_u8() as i32
}
}