Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-30 17:40:00 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: rm multisig db

Browse Source
This commit is contained in:
k
2024-08-05 11:47:52 +02:00
parent 9a8f5edd58
commit ce1fed8c16
16 changed files with 369 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
parser/src/databases/_database.rs
View File

@@ -15,53 +15,37 @@ use derive_deref::{Deref, DerefMut};
//
// Possible compression: https://pijul.org/posts/sanakirja-zstd/
use sanakirja::{
btree::{self, page, page_unsized, BTreeMutPage, Db_},
btree::{self, page, Db_},
direct_repr, Commit, Env, Error, MutTxn, RootDb, Storable, UnsizedStorable,
};
use crate::io::OUTPUTS_FOLDER_PATH;
pub type SizedDatabase<Key, Value> = Database<Key, Key, Value, page::Page<Key, Value>>;
pub type UnsizedDatabase<KeyTree, KeyDB, Value> =
Database<KeyTree, KeyDB, Value, page_unsized::Page<KeyDB, Value>>;
#[derive(Allocative)]
#[allocative(bound = "KeyTree: Allocative, KeyDB, Value: Allocative, Page")]
#[allocative(bound = "Key: Allocative, Value: Allocative")]
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way.
pub struct Database<KeyTree, KeyDB, Value, Page>
pub struct Database<Key, Value>
where
KeyTree: Ord + Clone + Debug,
KeyDB: Ord + ?Sized + Storable,
Key: Ord + Clone + Debug + ?Sized + Storable,
Value: Storable + PartialEq,
Page: BTreeMutPage<KeyDB, Value>,
{
pub cached_puts: BTreeMap<KeyTree, Value>,
pub cached_dels: BTreeSet<KeyTree>,
pub cached_puts: BTreeMap<Key, Value>,
pub cached_dels: BTreeSet<Key>,
#[allocative(skip)]
db: Db_<KeyDB, Value, Page>,
db: Db_<Key, Value, page::Page<Key, Value>>,
#[allocative(skip)]
txn: MutTxn<Env, ()>,
#[allocative(skip)]
key_tree_to_key_db: fn(&KeyTree) -> &KeyDB,
}
pub const SANAKIRJA_MAX_KEY_SIZE: usize = 510;
const ROOT_DB: usize = 0;
const PAGE_SIZE: u64 = 4096 * 256; // 1mo - Must be a multiplier of 4096
impl<KeyDB, KeyTree, Value, Page> Database<KeyTree, KeyDB, Value, Page>
impl<Key, Value> Database<Key, Value>
where
KeyTree: Ord + Clone + Debug,
KeyDB: Ord + ?Sized + Storable,
Key: Ord + Clone + Debug + ?Sized + Storable,
Value: Storable + PartialEq,
Page: BTreeMutPage<KeyDB, Value>,
{
pub fn open(
folder: &str,
file: &str,
key_tree_to_key_db: fn(&KeyTree) -> &KeyDB,
) -> color_eyre::Result<Self> {
pub fn open(folder: &str, file: &str) -> color_eyre::Result<Self> {
let mut txn = Self::init_txn(folder, file)?;
let db = txn
@@ -73,20 +57,19 @@ where
cached_dels: BTreeSet::default(),
db,
txn,
key_tree_to_key_db,
})
}
pub fn iter<F>(&self, callback: &mut F)
where
F: FnMut((&KeyDB, &Value)),
F: FnMut((&Key, &Value)),
{
btree::iter(&self.txn, &self.db, None)
.unwrap()
.for_each(|entry| callback(entry.unwrap()));
}
pub fn get(&self, key: &KeyTree) -> Option<&Value> {
pub fn get(&self, key: &Key) -> Option<&Value> {
if let Some(cached_put) = self.get_from_puts(key) {
return Some(cached_put);
}
@@ -94,13 +77,11 @@ where
self.db_get(key)
}
pub fn db_get(&self, key: &KeyTree) -> Option<&Value> {
let k = (self.key_tree_to_key_db)(key);
pub fn db_get(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
let option = btree::get(&self.txn, &self.db, k, None).unwrap();
if let Some((k_found, v)) = option {
if k == k_found {
if let Some((key_found, v)) = option {
if key == key_found {
return Some(v);
}
}
@@ -109,17 +90,17 @@ where
}
#[inline(always)]
pub fn get_from_puts(&self, key: &KeyTree) -> Option<&Value> {
pub fn get_from_puts(&self, key: &Key) -> Option<&Value> {
self.cached_puts.get(key)
}
#[inline(always)]
pub fn get_mut_from_puts(&mut self, key: &KeyTree) -> Option<&mut Value> {
pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> {
self.cached_puts.get_mut(key)
}
#[inline(always)]
pub fn remove(&mut self, key: &KeyTree) -> Option<Value> {
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.remove_from_puts(key).or_else(|| {
self.db_remove(key);
@@ -128,30 +109,30 @@ where
}
#[inline(always)]
pub fn db_remove(&mut self, key: &KeyTree) {
pub fn db_remove(&mut self, key: &Key) {
self.cached_dels.insert(key.clone());
}
pub fn update(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.insert(key.clone());
self.cached_puts.insert(key, value)
}
#[inline(always)]
pub fn remove_from_puts(&mut self, key: &KeyTree) -> Option<Value> {
pub fn remove_from_puts(&mut self, key: &Key) -> Option<Value> {
self.cached_puts.remove(key)
}
#[inline(always)]
pub fn insert(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.remove(&key);
self.unsafe_insert(key, value)
}
#[inline(always)]
pub fn unsafe_insert(&mut self, key: KeyTree, value: Value) -> Option<Value> {
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_puts.insert(key, value)
}
@@ -175,12 +156,7 @@ where
self.cached_dels
.into_iter()
.try_for_each(|key| -> Result<(), Error> {
btree::del(
&mut self.txn,
&mut self.db,
(self.key_tree_to_key_db)(&key),
None,
)?;
btree::del(&mut self.txn, &mut self.db, &key, None)?;
Ok(())
})?;
@@ -188,12 +164,7 @@ where
self.cached_puts
.into_iter()
.try_for_each(|(key, value)| -> Result<(), Error> {
btree::put(
&mut self.txn,
&mut self.db,
(self.key_tree_to_key_db)(&key),
&value,
)?;
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
})?;

6
parser/src/databases/address_index_to_address_data.rs
View File

@@ -12,11 +12,11 @@ use crate::{
utils::time,
};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = u32;
type Value = AddressData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct AddressIndexToAddressData {
@@ -78,7 +78,7 @@ impl AddressIndexToAddressData {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}

12
parser/src/databases/address_index_to_empty_address_data.rs
View File

@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Date, EmptyAddressData, Height};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = u32;
type Value = EmptyAddressData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct AddressIndexToEmptyAddressData {
@@ -45,12 +45,6 @@ impl AddressIndexToEmptyAddressData {
self.open_db(&key).unsafe_insert(key, value)
}
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
// self.metadata.called_remove();
// self.open_db(key).remove_from_puts(key)
// }
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.metadata.called_remove();
@@ -86,7 +80,7 @@ impl AddressIndexToEmptyAddressData {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}

35
parser/src/databases/address_to_address_index.rs
View File

@@ -5,16 +5,12 @@ use rayon::prelude::*;
use crate::structs::{Address, Date, Height};
use super::{
AnyDatabaseGroup, Database, Metadata, SizedDatabase, U8x19, U8x31,
UnsizedDatabase as _UnsizedDatabase,
};
use super::{AnyDatabaseGroup, Database, Metadata, U8x19, U8x31};
type Value = u32;
type U8x19Database = SizedDatabase<U8x19, Value>;
type U8x31Database = SizedDatabase<U8x31, Value>;
type U32Database = SizedDatabase<u32, Value>;
type UnsizedDatabase = _UnsizedDatabase<Box<[u8]>, [u8], Value>;
type U8x19Database = Database<U8x19, Value>;
type U8x31Database = Database<U8x31, Value>;
type U32Database = Database<u32, Value>;
type P2PKDatabase = U8x19Database;
type P2PKHDatabase = U8x19Database;
@@ -26,7 +22,7 @@ type UnknownDatabase = U32Database;
type OpReturnDatabase = U32Database;
type PushOnlyDatabase = U32Database;
type EmptyDatabase = U32Database;
type MultisigDatabase = UnsizedDatabase;
type MultisigDatabase = U32Database;
#[derive(Allocative)]
pub struct AddressToAddressIndex {
@@ -106,7 +102,7 @@ impl AddressToAddressIndex {
Address::Unknown(key) => self.unknown.as_ref().unwrap().get(key),
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get(key),
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get(key),
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get(key),
Address::MultiSig(key) => self.push_only.as_ref().unwrap().get(key),
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get(key),
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get(key),
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get(key),
@@ -155,7 +151,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2pk"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -166,7 +161,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2pkh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -177,7 +171,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2sh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -188,7 +181,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2wpkh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -199,7 +191,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2wsh"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -210,7 +201,6 @@ impl AddressToAddressIndex {
Database::open(
&format!("{}/{}", Self::folder(), "p2tr"),
&prefix.to_string(),
|key| key,
)
.unwrap()
})
@@ -218,28 +208,27 @@ impl AddressToAddressIndex {
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
self.unknown
.get_or_insert_with(|| Database::open(Self::folder(), "unknown", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "unknown").unwrap())
}
pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
self.op_return
.get_or_insert_with(|| Database::open(Self::folder(), "op_return", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "op_return").unwrap())
}
pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
self.push_only
.get_or_insert_with(|| Database::open(Self::folder(), "push_only", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "push_only").unwrap())
}
pub fn open_empty(&mut self) -> &mut UnknownDatabase {
self.empty
.get_or_insert_with(|| Database::open(Self::folder(), "empty", |key| key).unwrap())
.get_or_insert_with(|| Database::open(Self::folder(), "empty").unwrap())
}
pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
self.multisig.get_or_insert_with(|| {
Database::open(Self::folder(), "multisig", |key| key as &[u8]).unwrap()
})
self.multisig
.get_or_insert_with(|| Database::open(Self::folder(), "multisig").unwrap())
}
}

9
parser/src/databases/txid_to_tx_data.rs
View File

@@ -10,11 +10,11 @@ use rayon::prelude::*;
use crate::structs::{Date, Height, TxData};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase, U8x31};
use super::{AnyDatabaseGroup, Database as _Database, Metadata, U8x31};
type Key = U8x31;
type Value = TxData;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxidToTxData {
@@ -105,9 +105,8 @@ impl TxidToTxData {
pub fn open_db(&mut self, txid: &Txid) -> &mut Database {
let db_index = Self::db_index(txid);
self.entry(db_index).or_insert_with(|| {
SizedDatabase::open(Self::folder(), &db_index.to_string(), |key| key).unwrap()
})
self.entry(db_index)
.or_insert_with(|| Database::open(Self::folder(), &db_index.to_string()).unwrap())
}
fn txid_to_key(txid: &Txid) -> U8x31 {

6
parser/src/databases/txout_index_to_address_index.rs
View File

@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = u32;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxoutIndexToAddressIndex {
@@ -77,7 +77,7 @@ impl TxoutIndexToAddressIndex {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}

6
parser/src/databases/txout_index_to_amount.rs
View File

@@ -9,11 +9,11 @@ use rayon::prelude::*;
use crate::structs::{Amount, Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Metadata, SizedDatabase};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = Amount;
type Database = SizedDatabase<Key, Value>;
type Database = _Database<Key, Value>;
#[derive(Allocative)]
pub struct TxoutIndexToAmount {
@@ -77,7 +77,7 @@ impl TxoutIndexToAmount {
(db_index + 1) * DB_MAX_SIZE
);
SizedDatabase::open(Self::folder(), &db_name, |key| key).unwrap()
Database::open(Self::folder(), &db_name).unwrap()
})
}