Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-28 08:39:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: added databases defragmentation

Browse Source
This commit is contained in:
k
2024-10-30 19:13:41 +01:00
parent 6eaeca1f3d
commit e5d81b4d5c
29 changed files with 1869 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
parser/src/actions/iter_blocks.rs
View File

@@ -31,6 +31,14 @@ pub fn iter_blocks(
let mut databases = Databases::import();
if config.first_defragment() {
databases.defragment();
if true {
panic!("Done");
}
}
log("Imported databases");
let mut states = States::import().unwrap_or_default();
@@ -177,7 +185,11 @@ pub fn iter_blocks(
.as_ref()
.map_or(true, |date| date.is_first_of_month());
if is_check_point || height.is_close_to_end(approx_block_count) {
let ran_for_at_least_a_minute = instant.elapsed().as_secs() >= 60;
if (is_check_point && ran_for_at_least_a_minute)
|| height.is_close_to_end(approx_block_count)
{
break 'days;
}

117
parser/src/databases/_database.rs
View File

@@ -1,12 +1,13 @@
use std::{
collections::{BTreeMap, BTreeSet},
fmt::Debug,
fs,
fs, mem,
};
use allocative::Allocative;
use derive_deref::{Deref, DerefMut};
use itertools::Itertools;
// https://docs.rs/sanakirja/latest/sanakirja/index.html
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
//
@@ -31,6 +32,8 @@ where
{
pub cached_puts: BTreeMap<Key, Value>,
pub cached_dels: BTreeSet<Key>,
folder: String,
file: String,
#[allocative(skip)]
db: Db_<Key, Value, page::Page<Key, Value>>,
#[allocative(skip)]
@@ -53,6 +56,8 @@ where
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
Ok(Self {
folder: folder.to_owned(),
file: file.to_owned(),
cached_puts: BTreeMap::default(),
cached_dels: BTreeSet::default(),
db,
@@ -64,6 +69,16 @@ where
btree::iter(&self.txn, &self.db, None).unwrap()
}
fn iter_collect(&self) -> BTreeMap<Key, Value>
where
Value: Clone,
{
self.iter()
.map(|r| r.unwrap())
.map(|(key, value)| (key.clone(), value.clone()))
.collect::<_>()
}
pub fn get(&self, key: &Key) -> Option<&Value> {
if let Some(cached_put) = self.get_from_puts(key) {
return Some(cached_put);
@@ -72,6 +87,17 @@ where
self.db_get(key)
}
fn destroy(self) {
let path = self.path();
drop(self);
fs::remove_file(&path).unwrap_or_else(|_| {
dbg!(path);
panic!("Error");
});
}
pub fn db_get(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
@@ -114,6 +140,14 @@ where
self.cached_puts.insert(key, value)
}
fn len(&self) -> usize {
self.iter().try_len().unwrap_or_else(|e| e.0)
}
fn is_empty(&self) -> bool {
self.len() == 0
}
#[inline(always)]
pub fn remove_from_puts(&mut self, key: &Key) -> Option<Value> {
self.cached_puts.remove(key)
@@ -131,6 +165,10 @@ where
self.cached_puts.insert(key, value)
}
fn path(&self) -> String {
format!("{}/{}", databases_folder_path(&self.folder), self.file)
}
fn init_txn(folder: &str, file: &str) -> color_eyre::Result<MutTxn<Env, ()>> {
let path = databases_folder_path(folder);
@@ -143,31 +181,78 @@ where
Ok(txn)
}
pub fn export(mut self) -> color_eyre::Result<(), Error> {
fn db_multi_put(&mut self, tree: BTreeMap<Key, Value>) -> Result<(), Error> {
tree.into_iter()
.try_for_each(|(key, value)| -> Result<(), Error> {
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
})
}
fn db_multi_del(&mut self, tree: BTreeSet<Key>) -> Result<(), Error> {
tree.into_iter().try_for_each(|key| -> Result<(), Error> {
btree::del(&mut self.txn, &mut self.db, &key, None)?;
Ok(())
})
}
}
pub trait AnyDatabase {
fn export(self) -> color_eyre::Result<(), Error>;
fn boxed_export(self: Box<Self>) -> color_eyre::Result<(), Error>;
#[allow(unused)]
fn defragment(self);
fn boxed_defragment(self: Box<Self>);
}
impl<Key, Value> AnyDatabase for Database<Key, Value>
where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq + Clone,
{
fn export(self) -> color_eyre::Result<(), Error> {
Box::new(self).boxed_export()
}
fn boxed_export(mut self: Box<Self>) -> color_eyre::Result<(), Error> {
if self.cached_dels.is_empty() && self.cached_puts.is_empty() {
return Ok(());
}
self.cached_dels
.into_iter()
.try_for_each(|key| -> Result<(), Error> {
btree::del(&mut self.txn, &mut self.db, &key, None)?;
let cached_dels = mem::take(&mut self.cached_dels);
self.db_multi_del(cached_dels)?;
Ok(())
})?;
self.cached_puts
.into_iter()
.try_for_each(|(key, value)| -> Result<(), Error> {
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
})?;
let cached_puts = mem::take(&mut self.cached_puts);
self.db_multi_put(cached_puts)?;
self.txn.set_root(ROOT_DB, self.db.db.into());
self.txn.commit()
}
fn defragment(self) {
Box::new(self).boxed_defragment()
}
fn boxed_defragment(self: Box<Self>) {
let btree = self.iter_collect();
let folder = self.folder.to_owned();
let file = self.file.to_owned();
self.destroy();
let mut s = Self::open(&folder, &file).unwrap();
if !s.is_empty() {
dbg!(s.len());
panic!("Database isn't empty");
}
s.db_multi_put(btree).unwrap();
s.export().unwrap();
}
}
#[derive(

11
parser/src/databases/_trait.rs
View File

@@ -5,7 +5,7 @@ use crate::{
utils::log,
};
use super::databases_folder_path;
use super::{databases_folder_path, AnyDatabase};
pub trait AnyDatabaseGroup
where
@@ -13,10 +13,15 @@ where
{
fn import() -> Self;
fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()>;
fn folder<'a>() -> &'a str;
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>>;
fn open_all(&mut self);
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()>;
// fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()>;
// fn defragment(&mut self);
fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
log(&format!("Reset {}", Self::folder()));

69
parser/src/databases/address_index_to_address_data.rs
View File

@@ -5,6 +5,7 @@ use std::{
};
use allocative::Allocative;
use itertools::Itertools;
use rayon::prelude::*;
use crate::{
@@ -13,7 +14,7 @@ use crate::{
utils::time,
};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
type Key = u32;
type Value = AddressData;
@@ -97,18 +98,45 @@ impl AddressIndexToAddressData {
.iter()
.map(|r| r.unwrap().1)
.for_each(|address_data| s.increment(address_data).unwrap());
s
})
.sum()
})
}
pub fn open_all(&mut self) {
fn db_index(key: &Key) -> usize {
*key as usize / ADDRESS_INDEX_DB_MAX_SIZE
}
}
impl AnyDatabaseGroup for AddressIndexToAddressData {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 1),
map: BTreeMap::default(),
}
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
fn folder<'a>() -> &'a str {
"address_index_to_address_data"
}
fn open_all(&mut self) {
let path = Self::full_path();
fs::create_dir_all(&path).unwrap();
let folder = fs::read_dir(path);
fs::read_dir(path)
if folder.is_err() {
return;
}
folder
.unwrap()
.map(|entry| {
entry
@@ -126,35 +154,14 @@ impl AddressIndexToAddressData {
});
}
fn db_index(key: &Key) -> usize {
*key as usize / ADDRESS_INDEX_DB_MAX_SIZE
}
}
impl AnyDatabaseGroup for AddressIndexToAddressData {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 1),
map: BTreeMap::default(),
}
}
fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
mem::take(&mut self.map)
.into_par_iter()
.try_for_each(|(_, db)| db.export())?;
self.metadata.export(height, date).unwrap();
Ok(())
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
.collect_vec()
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
fn folder<'a>() -> &'a str {
"address_index_to_address_data"
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.metadata.export(height, date)
}
}

74
parser/src/databases/address_index_to_empty_address_data.rs
View File

@@ -1,15 +1,17 @@
use std::{
collections::BTreeMap,
mem,
fs, mem,
ops::{Deref, DerefMut},
};
use allocative::Allocative;
use rayon::prelude::*;
use itertools::Itertools;
use crate::structs::{Date, EmptyAddressData, Height};
use super::{AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE};
use super::{
AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE,
};
type Key = u32;
type Value = EmptyAddressData;
@@ -96,16 +98,6 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
}
}
fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
mem::take(&mut self.map)
.into_par_iter()
.try_for_each(|(_, db)| db.export())?;
self.metadata.export(height, date)?;
Ok(())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
@@ -113,4 +105,60 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
fn folder<'a>() -> &'a str {
"address_index_to_empty_address_data"
}
fn open_all(&mut self) {
let path = Self::full_path();
let folder = fs::read_dir(path);
if folder.is_err() {
return;
}
folder
.unwrap()
.map(|entry| {
entry
.unwrap()
.path()
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_owned()
})
.filter(|file_name| file_name.contains(".."))
.for_each(|path| {
self.open_db(&path.split("..").next().unwrap().parse::<u32>().unwrap());
});
}
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
mem::take(&mut self.map)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
.collect_vec()
}
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.metadata.export(height, date)
}
// fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
// self.drain_to_vec()
// .into_par_iter()
// .try_for_each(AnyDatabase::boxed_export)?;
// self.metadata.export(height, date)?;
// Ok(())
// }
// fn defragment(&mut self) {
// self.open_all();
// self.drain_to_vec()
// .into_par_iter()
// .for_each(AnyDatabase::boxed_defragment);
// }
}

284
parser/src/databases/address_to_address_index.rs
View File

@@ -1,11 +1,13 @@
use std::{collections::BTreeMap, mem, thread};
use std::{collections::BTreeMap, fs, mem};
use allocative::Allocative;
use rayon::prelude::*;
use itertools::Itertools;
use crate::structs::{Address, Date, Height};
use super::{AnyDatabaseGroup, Database, Metadata, U8x19, U8x31};
use super::{
databases_folder_path, AnyDatabase, AnyDatabaseGroup, Database, Metadata, U8x19, U8x31,
};
type Value = u32;
type U8x19Database = Database<U8x19, Value>;
@@ -146,64 +148,161 @@ impl AddressToAddressIndex {
}
}
pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase {
self.p2pk.entry(prefix).or_insert_with(|| {
Database::open(
&format!("{}/{}", Self::folder(), "p2pk"),
&prefix.to_string(),
)
fn path_to_group_prefixes(path: &str) -> Vec<u16> {
let path = databases_folder_path(path);
let folder = fs::read_dir(path);
if folder.is_err() {
return vec![];
}
folder
.unwrap()
})
.map(|entry| {
entry
.unwrap()
.path()
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_owned()
.parse::<u16>()
.unwrap()
})
.collect_vec()
}
fn path_p2pk() -> String {
format!("{}/{}", Self::folder(), "p2pk")
}
pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase {
let path = Self::path_p2pk();
self.p2pk
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
}
fn open_all_p2pk(&mut self) {
let path = Self::path_p2pk();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2pk
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
});
}
fn path_p2pkh() -> String {
format!("{}/{}", Self::folder(), "p2pkh")
}
pub fn open_p2pkh(&mut self, prefix: u16) -> &mut P2PKHDatabase {
self.p2pkh.entry(prefix).or_insert_with(|| {
Database::open(
&format!("{}/{}", Self::folder(), "p2pkh"),
&prefix.to_string(),
)
.unwrap()
})
let path = Self::path_p2pkh();
self.p2pkh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
}
fn open_all_p2pkh(&mut self) {
let path = Self::path_p2pkh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2pkh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
});
}
fn path_p2sh() -> String {
format!("{}/{}", Self::folder(), "p2sh")
}
pub fn open_p2sh(&mut self, prefix: u16) -> &mut P2SHDatabase {
self.p2sh.entry(prefix).or_insert_with(|| {
Database::open(
&format!("{}/{}", Self::folder(), "p2sh"),
&prefix.to_string(),
)
.unwrap()
})
let path = Self::path_p2sh();
self.p2sh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
}
fn open_all_p2sh(&mut self) {
let path = Self::path_p2sh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2sh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
});
}
fn path_p2wpkh() -> String {
format!("{}/{}", Self::folder(), "p2wpkh")
}
pub fn open_p2wpkh(&mut self, prefix: u16) -> &mut P2WPKHDatabase {
self.p2wpkh.entry(prefix).or_insert_with(|| {
Database::open(
&format!("{}/{}", Self::folder(), "p2wpkh"),
&prefix.to_string(),
)
.unwrap()
})
let path = Self::path_p2wpkh();
self.p2wpkh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
}
fn open_all_p2wpkh(&mut self) {
let path = Self::path_p2wpkh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2wpkh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
});
}
fn path_p2wsh() -> String {
format!("{}/{}", Self::folder(), "p2wsh")
}
pub fn open_p2wsh(&mut self, prefix: u16) -> &mut P2WSHDatabase {
self.p2wsh.entry(prefix).or_insert_with(|| {
Database::open(
&format!("{}/{}", Self::folder(), "p2wsh"),
&prefix.to_string(),
)
.unwrap()
})
let path = Self::path_p2wsh();
self.p2wsh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
}
fn open_all_p2wsh(&mut self) {
let path = Self::path_p2wsh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2wsh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
});
}
fn path_p2tr() -> String {
format!("{}/{}", Self::folder(), "p2tr")
}
pub fn open_p2tr(&mut self, prefix: u16) -> &mut P2TRDatabase {
self.p2tr.entry(prefix).or_insert_with(|| {
Database::open(
&format!("{}/{}", Self::folder(), "p2tr"),
&prefix.to_string(),
)
.unwrap()
})
let path = Self::path_p2tr();
self.p2tr
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
}
fn open_all_p2tr(&mut self) {
let path = Self::path_p2tr();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2tr
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
});
}
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
@@ -251,44 +350,6 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
}
}
fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
thread::scope(|s| {
s.spawn(|| {
mem::take(&mut self.p2pk)
.into_par_iter()
.chain(mem::take(&mut self.p2pkh).into_par_iter())
.chain(mem::take(&mut self.p2sh).into_par_iter())
.chain(mem::take(&mut self.p2wpkh).into_par_iter())
.try_for_each(|(_, db)| db.export())
});
s.spawn(|| {
mem::take(&mut self.p2wsh)
.into_par_iter()
.chain(mem::take(&mut self.p2tr).into_par_iter())
.try_for_each(|(_, db)| db.export())
});
s.spawn(|| {
[
self.unknown.take(),
self.op_return.take(),
self.push_only.take(),
self.empty.take(),
]
.into_par_iter()
.flatten()
.try_for_each(|db| db.export())
});
self.multisig.take().map(|db| db.export());
});
self.metadata.export(height, date)?;
Ok(())
}
fn reset_metadata(&mut self) {
self.metadata.reset()
}
@@ -296,4 +357,61 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
fn folder<'a>() -> &'a str {
"address_to_address_index"
}
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
mem::take(&mut self.p2pk)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
.chain(
mem::take(&mut self.p2pkh)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
)
.chain(
mem::take(&mut self.p2sh)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
)
.chain(
mem::take(&mut self.p2wpkh)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
)
.chain(
mem::take(&mut self.p2wsh)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
)
.chain(
mem::take(&mut self.p2tr)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
)
.chain(
[
self.unknown.take(),
self.op_return.take(),
self.push_only.take(),
self.empty.take(),
self.multisig.take(),
]
.into_iter()
.flatten()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
)
.collect_vec()
}
fn open_all(&mut self) {
self.open_all_p2pk();
self.open_all_p2pkh();
self.open_all_p2wpkh();
self.open_all_p2wsh();
self.open_all_p2sh();
self.open_all_p2tr();
}
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.metadata.export(height, date)
}
}

126
parser/src/databases/mod.rs
View File

@@ -17,12 +17,15 @@ use _trait::*;
pub use address_index_to_address_data::*;
pub use address_index_to_empty_address_data::*;
pub use address_to_address_index::*;
use itertools::Itertools;
use metadata::*;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
pub use txid_to_tx_data::*;
pub use txout_index_to_address_index::*;
pub use txout_index_to_amount::*;
use crate::{
log,
structs::{Date, Height},
utils::time,
};
@@ -61,51 +64,96 @@ impl Databases {
}
}
pub fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
self.txid_to_tx_data
.drain_to_vec()
.into_iter()
.chain(self.txout_index_to_amount.drain_to_vec())
.chain(self.address_to_address_index.drain_to_vec())
.chain(self.address_index_to_address_data.drain_to_vec())
.chain(self.address_index_to_empty_address_data.drain_to_vec())
.chain(self.txout_index_to_address_index.drain_to_vec())
.collect_vec()
}
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.txid_to_tx_data.export_metadata(height, date)?;
self.txout_index_to_amount.export_metadata(height, date)?;
self.address_index_to_address_data
.export_metadata(height, date)?;
self.address_index_to_empty_address_data
.export_metadata(height, date)?;
self.address_to_address_index
.export_metadata(height, date)?;
self.txout_index_to_address_index
.export_metadata(height, date)?;
Ok(())
}
pub fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
thread::scope(|s| {
s.spawn(|| {
time("> Database txid_to_tx_data", || {
self.txid_to_tx_data.export(height, date)
})
});
self.export_metadata(height, date)?;
s.spawn(|| {
time("> Database txout_index_to_amount", || {
self.txout_index_to_amount.export(height, date)
})
});
});
thread::scope(|s| {
s.spawn(|| {
time("> Database address_index_to_address_data", || {
self.address_index_to_address_data.export(height, date)
})
});
s.spawn(|| {
time("> Database address_index_to_empty_address_data", || {
self.address_index_to_empty_address_data
.export(height, date)
})
});
s.spawn(|| {
time("> Database address_to_address_index", || {
self.address_to_address_index.export(height, date)
})
});
s.spawn(|| {
time("> Database txout_index_to_address_index", || {
self.txout_index_to_address_index.export(height, date)
})
});
});
self.drain_to_vec()
.into_par_iter()
.try_for_each(AnyDatabase::boxed_export)?;
Ok(())
}
fn open_all(&mut self) {
thread::scope(|s| {
s.spawn(|| {
time("Opening all address_index_to_address_data", || {
self.address_index_to_address_data.open_all()
})
});
s.spawn(|| {
time("Opening all address_index_to_empty_address_data", || {
self.address_index_to_empty_address_data.open_all()
})
});
s.spawn(|| {
time("Opening all address_to_address_index", || {
self.address_to_address_index.open_all()
})
});
s.spawn(|| {
time("Opening all txid_to_tx_data", || {
self.txid_to_tx_data.open_all()
})
});
s.spawn(|| {
time("Opening all txout_index_to_address_index", || {
self.txout_index_to_address_index.open_all()
})
});
s.spawn(|| {
time("Opening all txout_index_to_amount", || {
self.txout_index_to_amount.open_all()
})
});
});
}
pub fn defragment(&mut self) {
log("Databases defragmentation");
time("Defragmenting databases", || {
time("Opened all databases", || self.open_all());
log("Defragmenting...");
self.drain_to_vec()
.into_par_iter()
.for_each(AnyDatabase::boxed_defragment);
})
}
pub fn reset(&mut self, include_addresses: bool) {
if include_addresses {
let _ = self.address_index_to_address_data.reset();

58
parser/src/databases/txid_to_tx_data.rs
View File

@@ -1,16 +1,16 @@
use std::{
collections::BTreeMap,
mem,
fs, mem,
ops::{Deref, DerefMut},
};
use allocative::Allocative;
use biter::bitcoin::Txid;
use rayon::prelude::*;
use itertools::Itertools;
use crate::structs::{Date, Height, TxData};
use super::{AnyDatabaseGroup, Database as _Database, Metadata, U8x31};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, U8x31};
type Key = U8x31;
type Value = TxData;
@@ -104,7 +104,11 @@ impl TxidToTxData {
#[inline(always)]
pub fn open_db(&mut self, txid: &Txid) -> &mut Database {
let db_index = Self::db_index(txid);
self._open_db(db_index)
}
#[inline(always)]
pub fn _open_db(&mut self, db_index: u16) -> &mut Database {
self.entry(db_index)
.or_insert_with(|| Database::open(Self::folder(), &db_index.to_string()).unwrap())
}
@@ -127,16 +131,6 @@ impl AnyDatabaseGroup for TxidToTxData {
}
}
fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
mem::take(&mut self.map)
.into_par_iter()
.try_for_each(|(_, db)| db.export())?;
self.metadata.export(height, date)?;
Ok(())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
@@ -144,4 +138,42 @@ impl AnyDatabaseGroup for TxidToTxData {
fn folder<'a>() -> &'a str {
"txid_to_tx_data"
}
fn open_all(&mut self) {
let path = Self::full_path();
let folder = fs::read_dir(path);
if folder.is_err() {
return;
}
folder
.unwrap()
.flat_map(|entry| {
entry
.unwrap()
.path()
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_owned()
.parse::<u16>()
})
.for_each(|db_index| {
self._open_db(db_index);
});
}
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
mem::take(&mut self.map)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
.collect_vec()
}
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.metadata.export(height, date)
}
}

66
parser/src/databases/txout_index_to_address_index.rs
View File

@@ -1,15 +1,15 @@
use std::{
collections::BTreeMap,
mem,
fs, mem,
ops::{Deref, DerefMut},
};
use allocative::Allocative;
use rayon::prelude::*;
use itertools::Itertools;
use crate::structs::{Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = u32;
@@ -95,15 +95,15 @@ impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
}
}
fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
mem::take(&mut self.map)
.into_par_iter()
.try_for_each(|(_, db)| db.export())?;
// fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
// mem::take(&mut self.map)
// .into_par_iter()
// .try_for_each(|(_, db)| db.export())?;
self.metadata.export(height, date)?;
// self.metadata.export(height, date)?;
Ok(())
}
// Ok(())
// }
fn reset_metadata(&mut self) {
self.metadata.reset();
@@ -112,4 +112,50 @@ impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
fn folder<'a>() -> &'a str {
"txout_index_to_address_index"
}
fn open_all(&mut self) {
let path = Self::full_path();
let folder = fs::read_dir(path);
if folder.is_err() {
return;
}
folder
.unwrap()
.map(|entry| {
entry
.unwrap()
.path()
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_owned()
})
.filter(|file_name| file_name.contains(".."))
.for_each(|path| {
self.open_db(
&path
.split("..")
.next()
.unwrap()
.parse::<u64>()
.unwrap()
.into(),
);
});
}
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
mem::take(&mut self.map)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
.collect_vec()
}
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.metadata.export(height, date)
}
}

62
parser/src/databases/txout_index_to_amount.rs
View File

@@ -1,15 +1,15 @@
use std::{
collections::BTreeMap,
mem,
fs, mem,
ops::{Deref, DerefMut},
};
use allocative::Allocative;
use rayon::prelude::*;
use itertools::Itertools;
use crate::structs::{Amount, Date, Height, TxoutIndex};
use super::{AnyDatabaseGroup, Database as _Database, Metadata};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
type Key = TxoutIndex;
type Value = Amount;
@@ -95,16 +95,6 @@ impl AnyDatabaseGroup for TxoutIndexToAmount {
}
}
fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
mem::take(&mut self.map)
.into_par_iter()
.try_for_each(|(_, db)| db.export())?;
self.metadata.export(height, date)?;
Ok(())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
@@ -112,4 +102,50 @@ impl AnyDatabaseGroup for TxoutIndexToAmount {
fn folder<'a>() -> &'a str {
"txout_index_to_amount"
}
fn open_all(&mut self) {
let path = Self::full_path();
let folder = fs::read_dir(path);
if folder.is_err() {
return;
}
folder
.unwrap()
.map(|entry| {
entry
.unwrap()
.path()
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_owned()
})
.filter(|file_name| file_name.contains(".."))
.for_each(|path| {
self.open_db(
&path
.split("..")
.next()
.unwrap()
.parse::<u64>()
.unwrap()
.into(),
);
});
}
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
mem::take(&mut self.map)
.into_values()
.map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
.collect_vec()
}
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.metadata.export(height, date)
}
}

13
parser/src/structs/config.rs
View File

@@ -32,10 +32,9 @@ pub struct Config {
#[arg(long, value_name = "SECONDS")]
pub delay: Option<u64>,
/// Maximum ram you want the program to use in GB, default: 50% of total, not saved
// Maximum ram you want the program to use in GB, default: 50% of total, not saved
// #[arg(long, value_name = "GB")]
// pub max_ram: Option<f64>,
/// Start a dry run, default: false, not saved
#[arg(long, value_name = "BOOL")]
dry_run: Option<bool>,
@@ -47,6 +46,10 @@ pub struct Config {
/// Recompute all computed datasets, default: false, not saved
#[arg(long, value_name = "BOOL")]
recompute_computed: Option<bool>,
/// Start the program by defragmenting all databases to reduce their footprint, default: false, not saved
#[arg(long, value_name = "BOOL")]
first_defragment: Option<bool>,
}
impl Config {
@@ -99,6 +102,7 @@ impl Config {
config.dry_run = config_args.dry_run.take();
config.record_ram_usage = config_args.record_ram_usage.take();
config.recompute_computed = config_args.recompute_computed.take();
config.first_defragment = config_args.first_defragment.take();
log("---");
log("Configuration:");
@@ -115,6 +119,7 @@ impl Config {
"recompute_computed: {:?}",
config.recompute_computed
));
log(&format!("first_defragment: {:?}", config.first_defragment));
log("---");
if config_args != Config::default() {
@@ -162,4 +167,8 @@ impl Config {
pub fn recompute_computed(&self) -> bool {
self.recompute_computed.is_some_and(|b| b)
}
pub fn first_defragment(&self) -> bool {
self.first_defragment.is_some_and(|b| b)
}
}

14
parser/src/structs/txout_index.rs
View File

@@ -9,6 +9,9 @@ pub struct TxoutIndex {
}
direct_repr!(TxoutIndex);
const SHIFT: u64 = 16;
const AND: u64 = (1 << SHIFT) - 1;
impl TxoutIndex {
#[inline(always)]
pub fn new(tx_index: u32, vout: u16) -> Self {
@@ -17,6 +20,15 @@ impl TxoutIndex {
#[inline(always)]
pub fn as_u64(&self) -> u64 {
((self.tx_index as u64) << 16_u64) + self.vout as u64
((self.tx_index as u64) << SHIFT) + self.vout as u64
}
}
impl From<u64> for TxoutIndex {
fn from(value: u64) -> Self {
Self {
tx_index: (value >> SHIFT) as u32,
vout: (value & AND) as u16,
}
}
}