brk: first commit

_src/parser/databases/_trait.rs

@@ -0,0 +1,60 @@
+use std::{fs, io, path::Path};
+
+use log::info;
+use snkrj::AnyDatabase;
+
+use crate::structs::{Config, Date, Height};
+
+use super::Metadata;
+
+pub trait AnyDatabaseGroup
+where
+    Self: Sized,
+{
+    fn init(config: &Config) -> Self {
+        let s = Self::import(config);
+        s.create_dir_all().unwrap();
+        s
+    }
+
+    fn import(config: &Config) -> Self;
+
+    fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>>;
+    fn open_all(&mut self);
+
+    fn metadata(&mut self) -> &mut Metadata;
+    fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
+        self.metadata().export(height, date)
+    }
+
+    fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
+        fs::create_dir_all(self.path())
+    }
+
+    fn remove_dir_all(&self) -> color_eyre::Result<(), io::Error> {
+        fs::remove_dir_all(self.path())
+    }
+
+    fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
+        info!(
+            "Reset {}",
+            self.path()
+                .components()
+                .last()
+                .unwrap()
+                .as_os_str()
+                .to_str()
+                .unwrap()
+        );
+
+        self.reset_metadata();
+        self.remove_dir_all()?;
+        self.create_dir_all()?;
+
+        Ok(())
+    }
+
+    fn reset_metadata(&mut self);
+
+    fn path(&self) -> &Path;
+}

_src/parser/databases/address_index_to_address_data.rs

@@ -0,0 +1,173 @@
+use std::{
+    collections::BTreeMap,
+    fs, mem,
+    ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
+};
+
+use allocative::Allocative;
+use itertools::Itertools;
+use rayon::prelude::*;
+use snkrj::{AnyDatabase, Database as _Database};
+
+use crate::{
+    parser::states::AddressCohortsDurableStates,
+    structs::{AddressData, Config},
+};
+
+use super::{AnyDatabaseGroup, Metadata};
+
+type Key = u32;
+type Value = AddressData;
+type Database = _Database<Key, Value>;
+
+#[derive(Allocative)]
+pub struct AddressIndexToAddressData {
+    path: PathBuf,
+    pub metadata: Metadata,
+    #[allocative(skip)]
+    pub map: BTreeMap<usize, Database>,
+}
+
+impl Deref for AddressIndexToAddressData {
+    type Target = BTreeMap<usize, Database>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.map
+    }
+}
+
+impl DerefMut for AddressIndexToAddressData {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.map
+    }
+}
+
+pub const ADDRESS_INDEX_DB_MAX_SIZE: usize = 250_000;
+
+impl AddressIndexToAddressData {
+    pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
+        self.metadata.called_insert();
+
+        self.open_db(&key).insert_to_ram(key, value)
+    }
+
+    pub fn remove(&mut self, key: &Key) -> Option<Value> {
+        self.metadata.called_remove();
+
+        self.open_db(key).remove(key)
+    }
+
+    /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
+    /// Though it makes it easy to use with rayon.
+    pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
+        let db_index = Self::db_index(key);
+
+        self.get(&db_index).unwrap().get_from_ram(key)
+    }
+
+    pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
+        let db_index = Self::db_index(key);
+
+        self.get(&db_index).unwrap().get_from_disk(key)
+    }
+
+    pub fn open_db(&mut self, key: &Key) -> &mut Database {
+        let db_index = Self::db_index(key);
+        let path = self.path().to_owned();
+
+        self.entry(db_index).or_insert_with(|| {
+            let db_name = format!(
+                "{}..{}",
+                db_index * ADDRESS_INDEX_DB_MAX_SIZE,
+                (db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
+            );
+
+            let path = path.join(db_name);
+
+            Database::open(path).unwrap()
+        })
+    }
+
+    pub fn compute_addres_cohorts_durable_states(&mut self) -> AddressCohortsDurableStates {
+        // time("Iter through address_index_to_address_data", || {
+        self.open_all();
+
+        // MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
+        mem::take(&mut self.map)
+            .par_iter()
+            .map(|(_, database)| {
+                let mut s = AddressCohortsDurableStates::default();
+
+                database
+                    .iter_disk()
+                    .map(|r| r.unwrap().1)
+                    .for_each(|address_data| s.increment(address_data).unwrap());
+
+                s
+            })
+            .sum()
+        // })
+    }
+
+    fn db_index(key: &Key) -> usize {
+        *key as usize / ADDRESS_INDEX_DB_MAX_SIZE
+    }
+}
+
+impl AnyDatabaseGroup for AddressIndexToAddressData {
+    fn import(config: &Config) -> Self {
+        let path = config
+            .path_databases()
+            .join("address_index_to_address_data");
+        Self {
+            metadata: Metadata::import(&path, 1),
+            path,
+            map: BTreeMap::default(),
+        }
+    }
+
+    fn reset_metadata(&mut self) {
+        self.metadata.reset();
+    }
+
+    fn open_all(&mut self) {
+        let folder = fs::read_dir(&self.path);
+
+        if folder.is_err() {
+            return;
+        }
+
+        folder
+            .unwrap()
+            .map(|entry| {
+                entry
+                    .unwrap()
+                    .path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_owned()
+            })
+            .filter(|file_name| file_name.contains(".."))
+            .for_each(|path| {
+                self.open_db(&path.split("..").next().unwrap().parse::<u32>().unwrap());
+            });
+    }
+
+    fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
+        mem::take(&mut self.map)
+            .into_values()
+            .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
+            .collect_vec()
+    }
+
+    fn metadata(&mut self) -> &mut Metadata {
+        &mut self.metadata
+    }
+
+    fn path(&self) -> &Path {
+        &self.path
+    }
+}

_src/parser/databases/address_index_to_empty_address_data.rs

@@ -0,0 +1,151 @@
+use std::{
+    collections::BTreeMap,
+    fs, mem,
+    ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
+};
+
+use allocative::Allocative;
+use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
+
+use crate::structs::{Config, EmptyAddressData};
+
+use super::{AnyDatabaseGroup, Metadata, ADDRESS_INDEX_DB_MAX_SIZE};
+
+type Key = u32;
+type Value = EmptyAddressData;
+type Database = _Database<Key, Value>;
+
+#[derive(Allocative)]
+pub struct AddressIndexToEmptyAddressData {
+    path: PathBuf,
+    pub metadata: Metadata,
+    #[allocative(skip)]
+    map: BTreeMap<usize, Database>,
+}
+
+impl Deref for AddressIndexToEmptyAddressData {
+    type Target = BTreeMap<usize, Database>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.map
+    }
+}
+
+impl DerefMut for AddressIndexToEmptyAddressData {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.map
+    }
+}
+
+impl AddressIndexToEmptyAddressData {
+    pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
+        self.metadata.called_insert();
+
+        self.open_db(&key).insert_to_ram(key, value)
+    }
+
+    pub fn remove(&mut self, key: &Key) -> Option<Value> {
+        self.metadata.called_remove();
+
+        self.open_db(key).remove(key)
+    }
+
+    /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
+    /// Though it makes it easy to use with rayon.
+    pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
+        let db_index = Self::db_index(key);
+
+        self.get(&db_index).and_then(|db| db.get_from_ram(key))
+    }
+
+    pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
+        let db_index = Self::db_index(key);
+
+        self.get(&db_index)
+            .unwrap_or_else(|| {
+                dbg!(&self.map.keys(), &key, &db_index);
+                panic!()
+            })
+            .get_from_disk(key)
+    }
+
+    pub fn open_db(&mut self, key: &Key) -> &mut Database {
+        let db_index = Self::db_index(key);
+        let path = self.path.to_owned();
+
+        self.entry(db_index).or_insert_with(|| {
+            let db_name = format!(
+                "{}..{}",
+                db_index * ADDRESS_INDEX_DB_MAX_SIZE,
+                (db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
+            );
+
+            let path = path.join(db_name);
+
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn db_index(key: &Key) -> usize {
+        *key as usize / ADDRESS_INDEX_DB_MAX_SIZE
+    }
+}
+
+impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
+    fn import(config: &Config) -> Self {
+        let path = config
+            .path_databases()
+            .join("address_index_to_empty_address_data");
+        Self {
+            metadata: Metadata::import(&path, 1),
+            path,
+            map: BTreeMap::default(),
+        }
+    }
+
+    fn reset_metadata(&mut self) {
+        self.metadata.reset();
+    }
+
+    fn open_all(&mut self) {
+        let folder = fs::read_dir(&self.path);
+
+        if folder.is_err() {
+            return;
+        }
+
+        folder
+            .unwrap()
+            .map(|entry| {
+                entry
+                    .unwrap()
+                    .path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_owned()
+            })
+            .filter(|file_name| file_name.contains(".."))
+            .for_each(|path| {
+                self.open_db(&path.split("..").next().unwrap().parse::<u32>().unwrap());
+            });
+    }
+
+    fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
+        mem::take(&mut self.map)
+            .into_values()
+            .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
+            .collect_vec()
+    }
+
+    fn metadata(&mut self) -> &mut Metadata {
+        &mut self.metadata
+    }
+
+    fn path(&self) -> &Path {
+        &self.path
+    }
+}

_src/parser/databases/address_to_address_index.rs

@@ -0,0 +1,445 @@
+use std::{
+    collections::BTreeMap,
+    fs, mem,
+    path::{Path, PathBuf},
+};
+
+use allocative::Allocative;
+use itertools::Itertools;
+use snkrj::{AnyDatabase, Database};
+
+use crate::structs::{Address, Config, U8x19, U8x31};
+
+use super::{AnyDatabaseGroup, Metadata};
+
+type Value = u32;
+type U8x19Database = Database<U8x19, Value>;
+type U8x31Database = Database<U8x31, Value>;
+type U32Database = Database<u32, Value>;
+
+type P2PKDatabase = U8x19Database;
+type P2PKHDatabase = U8x19Database;
+type P2SHDatabase = U8x19Database;
+type P2WPKHDatabase = U8x19Database;
+type P2WSHDatabase = U8x31Database;
+type P2TRDatabase = U8x31Database;
+type UnknownDatabase = U32Database;
+type OpReturnDatabase = U32Database;
+type PushOnlyDatabase = U32Database;
+type EmptyDatabase = U32Database;
+type MultisigDatabase = U32Database;
+
+#[derive(Allocative)]
+pub struct AddressToAddressIndex {
+    path: PathBuf,
+    pub metadata: Metadata,
+
+    #[allocative(skip)]
+    p2pk: BTreeMap<u16, P2PKDatabase>,
+    #[allocative(skip)]
+    p2pkh: BTreeMap<u16, P2PKHDatabase>,
+    #[allocative(skip)]
+    p2sh: BTreeMap<u16, P2SHDatabase>,
+    #[allocative(skip)]
+    p2wpkh: BTreeMap<u16, P2WPKHDatabase>,
+    #[allocative(skip)]
+    p2wsh: BTreeMap<u16, P2WSHDatabase>,
+    #[allocative(skip)]
+    p2tr: BTreeMap<u16, P2TRDatabase>,
+    #[allocative(skip)]
+    op_return: Option<OpReturnDatabase>,
+    #[allocative(skip)]
+    push_only: Option<PushOnlyDatabase>,
+    #[allocative(skip)]
+    unknown: Option<UnknownDatabase>,
+    #[allocative(skip)]
+    empty: Option<EmptyDatabase>,
+    #[allocative(skip)]
+    multisig: Option<MultisigDatabase>,
+}
+
+impl AddressToAddressIndex {
+    pub fn open_db(&mut self, address: &Address) {
+        match address {
+            Address::Empty(_) => {
+                self.open_empty();
+            }
+            Address::Unknown(_) => {
+                self.open_unknown();
+            }
+            Address::OpReturn(_) => {
+                self.open_op_return();
+            }
+            Address::PushOnly(_) => {
+                self.open_push_only();
+            }
+            Address::MultiSig(_) => {
+                self.open_multisig();
+            }
+            Address::P2PK((prefix, _)) => {
+                self.open_p2pk(*prefix);
+            }
+            Address::P2PKH((prefix, _)) => {
+                self.open_p2pkh(*prefix);
+            }
+            Address::P2SH((prefix, _)) => {
+                self.open_p2sh(*prefix);
+            }
+            Address::P2WPKH((prefix, _)) => {
+                self.open_p2wpkh(*prefix);
+            }
+            Address::P2WSH((prefix, _)) => {
+                self.open_p2wsh(*prefix);
+            }
+            Address::P2TR((prefix, _)) => {
+                self.open_p2tr(*prefix);
+            }
+        }
+    }
+
+    /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed.
+    /// Though it makes it easy to use with rayon
+    pub fn unsafe_get(&self, address: &Address) -> Option<&Value> {
+        match address {
+            Address::Empty(key) => self.empty.as_ref().unwrap().get(key),
+            Address::Unknown(key) => self.unknown.as_ref().unwrap().get(key),
+            Address::OpReturn(key) => self.op_return.as_ref().unwrap().get(key),
+            Address::PushOnly(key) => self.push_only.as_ref().unwrap().get(key),
+            Address::MultiSig(key) => self.multisig.as_ref().unwrap().get(key),
+            Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get(key),
+            Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get(key),
+            Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get(key),
+            Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get(key),
+            Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get(key),
+            Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get(key),
+        }
+    }
+
+    pub fn get_from_ram(&self, address: &Address) -> Option<&Value> {
+        match address {
+            Address::Empty(key) => self.empty.as_ref().unwrap().get_from_ram(key),
+            Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_ram(key),
+            Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_ram(key),
+            Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_ram(key),
+            Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_ram(key),
+            Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_ram(key),
+            Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_ram(key),
+            Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_ram(key),
+            Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_ram(key),
+            Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_ram(key),
+            Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_ram(key),
+        }
+    }
+
+    pub fn insert(&mut self, address: Address, value: Value) -> Option<Value> {
+        self.metadata.called_insert();
+
+        match address {
+            Address::Empty(key) => self.open_empty().insert(key, value),
+            Address::Unknown(key) => self.open_unknown().insert(key, value),
+            Address::OpReturn(key) => self.open_op_return().insert(key, value),
+            Address::PushOnly(key) => self.open_push_only().insert(key, value),
+            Address::MultiSig(key) => self.open_multisig().insert(key, value),
+            Address::P2PK((prefix, rest)) => self.open_p2pk(prefix).insert(rest, value),
+            Address::P2PKH((prefix, rest)) => self.open_p2pkh(prefix).insert(rest, value),
+            Address::P2SH((prefix, rest)) => self.open_p2sh(prefix).insert(rest, value),
+            Address::P2WPKH((prefix, rest)) => self.open_p2wpkh(prefix).insert(rest, value),
+            Address::P2WSH((prefix, rest)) => self.open_p2wsh(prefix).insert(rest, value),
+            Address::P2TR((prefix, rest)) => self.open_p2tr(prefix).insert(rest, value),
+        }
+    }
+
+    fn path_to_group_prefixes(path: &Path) -> Vec<u16> {
+        let folder = fs::read_dir(path);
+
+        if folder.is_err() {
+            return vec![];
+        }
+
+        folder
+            .unwrap()
+            .map(|entry| {
+                entry
+                    .unwrap()
+                    .path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_owned()
+                    .parse::<u16>()
+                    .unwrap()
+            })
+            .collect_vec()
+    }
+
+    fn path_p2pk(&self) -> PathBuf {
+        self.path().join("p2pk")
+    }
+
+    pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase {
+        let path = self.path_p2pk();
+        self.p2pk.entry(prefix).or_insert_with(|| {
+            let path = path.join(prefix.to_string());
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn open_all_p2pk(&mut self) {
+        let path = self.path_p2pk();
+        Self::path_to_group_prefixes(&path)
+            .into_iter()
+            .for_each(|prefix| {
+                self.p2pk.insert(prefix, {
+                    let path = path.join(prefix.to_string());
+                    Database::open(path).unwrap()
+                });
+            });
+    }
+
+    fn path_p2pkh(&self) -> PathBuf {
+        self.path().join("p2pkh")
+    }
+
+    pub fn open_p2pkh(&mut self, prefix: u16) -> &mut P2PKHDatabase {
+        let path = self.path_p2pkh();
+
+        self.p2pkh.entry(prefix).or_insert_with(|| {
+            let path = path.join(prefix.to_string());
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn open_all_p2pkh(&mut self) {
+        let path = self.path_p2pkh();
+        Self::path_to_group_prefixes(&path)
+            .into_iter()
+            .for_each(|prefix| {
+                self.p2pkh.insert(prefix, {
+                    let path = path.join(prefix.to_string());
+                    Database::open(path).unwrap()
+                });
+            });
+    }
+
+    fn path_p2sh(&self) -> PathBuf {
+        self.path().join("p2sh")
+    }
+
+    pub fn open_p2sh(&mut self, prefix: u16) -> &mut P2SHDatabase {
+        let path = self.path_p2sh();
+
+        self.p2sh.entry(prefix).or_insert_with(|| {
+            let path = path.join(prefix.to_string());
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn open_all_p2sh(&mut self) {
+        let path = self.path_p2sh();
+        Self::path_to_group_prefixes(&path)
+            .into_iter()
+            .for_each(|prefix| {
+                self.p2sh.insert(prefix, {
+                    let path = path.join(prefix.to_string());
+                    Database::open(path).unwrap()
+                });
+            });
+    }
+
+    fn path_p2wpkh(&self) -> PathBuf {
+        self.path().join("p2wpkh")
+    }
+
+    pub fn open_p2wpkh(&mut self, prefix: u16) -> &mut P2WPKHDatabase {
+        let path = self.path_p2wpkh();
+
+        self.p2wpkh.entry(prefix).or_insert_with(|| {
+            let path = path.join(prefix.to_string());
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn open_all_p2wpkh(&mut self) {
+        let path = self.path_p2wpkh();
+        Self::path_to_group_prefixes(&path)
+            .into_iter()
+            .for_each(|prefix| {
+                self.p2wpkh.insert(prefix, {
+                    let path = path.join(prefix.to_string());
+                    Database::open(path).unwrap()
+                });
+            });
+    }
+
+    fn path_p2wsh(&self) -> PathBuf {
+        self.path().join("p2wsh")
+    }
+
+    pub fn open_p2wsh(&mut self, prefix: u16) -> &mut P2WSHDatabase {
+        let path = self.path_p2wsh();
+
+        self.p2wsh.entry(prefix).or_insert_with(|| {
+            let path = path.join(prefix.to_string());
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn open_all_p2wsh(&mut self) {
+        let path = self.path_p2wsh();
+        Self::path_to_group_prefixes(&path)
+            .into_iter()
+            .for_each(|prefix| {
+                self.p2wsh.insert(prefix, {
+                    let path = path.join(prefix.to_string());
+                    Database::open(path).unwrap()
+                });
+            });
+    }
+
+    fn path_p2tr(&self) -> PathBuf {
+        self.path().join("p2tr")
+    }
+
+    pub fn open_p2tr(&mut self, prefix: u16) -> &mut P2TRDatabase {
+        let path = self.path_p2tr();
+
+        self.p2tr.entry(prefix).or_insert_with(|| {
+            let path = path.join(prefix.to_string());
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn open_all_p2tr(&mut self) {
+        let path = self.path_p2tr();
+        Self::path_to_group_prefixes(&path)
+            .into_iter()
+            .for_each(|prefix| {
+                self.p2tr.insert(prefix, {
+                    let path = path.join(prefix.to_string());
+                    Database::open(path).unwrap()
+                });
+            });
+    }
+
+    pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
+        self.unknown
+            .get_or_insert_with(|| Database::open(self.path.join("unknown")).unwrap())
+    }
+
+    pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
+        self.op_return
+            .get_or_insert_with(|| Database::open(self.path.join("op_return")).unwrap())
+    }
+
+    pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
+        self.push_only
+            .get_or_insert_with(|| Database::open(self.path.join("push_only")).unwrap())
+    }
+
+    pub fn open_empty(&mut self) -> &mut UnknownDatabase {
+        self.empty
+            .get_or_insert_with(|| Database::open(self.path.join("empty")).unwrap())
+    }
+
+    pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
+        self.multisig
+            .get_or_insert_with(|| Database::open(self.path.join("multisig")).unwrap())
+    }
+}
+
+impl AnyDatabaseGroup for AddressToAddressIndex {
+    fn import(config: &Config) -> Self {
+        let path = config.path_databases().join("address_to_address_index");
+        Self {
+            metadata: Metadata::import(&path, 1),
+            path,
+
+            p2pk: BTreeMap::default(),
+            p2pkh: BTreeMap::default(),
+            p2sh: BTreeMap::default(),
+            p2wpkh: BTreeMap::default(),
+            p2wsh: BTreeMap::default(),
+            p2tr: BTreeMap::default(),
+            op_return: None,
+            push_only: None,
+            unknown: None,
+            empty: None,
+            multisig: None,
+        }
+    }
+
+    fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
+        fs::create_dir_all(self.path_p2pk()).unwrap();
+        fs::create_dir_all(self.path_p2pkh()).unwrap();
+        fs::create_dir_all(self.path_p2sh()).unwrap();
+        fs::create_dir_all(self.path_p2wpkh()).unwrap();
+        fs::create_dir_all(self.path_p2wsh()).unwrap();
+        fs::create_dir_all(self.path_p2tr())
+    }
+
+    fn reset_metadata(&mut self) {
+        self.metadata.reset()
+    }
+
+    fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
+        mem::take(&mut self.p2pk)
+            .into_values()
+            .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
+            .chain(
+                mem::take(&mut self.p2pkh)
+                    .into_values()
+                    .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
+            )
+            .chain(
+                mem::take(&mut self.p2sh)
+                    .into_values()
+                    .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
+            )
+            .chain(
+                mem::take(&mut self.p2wpkh)
+                    .into_values()
+                    .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
+            )
+            .chain(
+                mem::take(&mut self.p2wsh)
+                    .into_values()
+                    .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
+            )
+            .chain(
+                mem::take(&mut self.p2tr)
+                    .into_values()
+                    .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
+            )
+            .chain(
+                [
+                    self.unknown.take(),
+                    self.op_return.take(),
+                    self.push_only.take(),
+                    self.empty.take(),
+                    self.multisig.take(),
+                ]
+                .into_iter()
+                .flatten()
+                .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>),
+            )
+            .collect_vec()
+    }
+
+    fn open_all(&mut self) {
+        self.open_all_p2pk();
+        self.open_all_p2pkh();
+        self.open_all_p2wpkh();
+        self.open_all_p2wsh();
+        self.open_all_p2sh();
+        self.open_all_p2tr();
+    }
+
+    fn metadata(&mut self) -> &mut Metadata {
+        &mut self.metadata
+    }
+
+    fn path(&self) -> &Path {
+        &self.path
+    }
+}

_src/parser/databases/metadata.rs

@@ -0,0 +1,127 @@
+use allocative::Allocative;
+use bincode::{Decode, Encode};
+use color_eyre::eyre::eyre;
+use serde::{Deserialize, Serialize};
+use std::{
+    fmt::Debug,
+    fs, io,
+    ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
+};
+
+use crate::{
+    io::Serialization,
+    structs::{Counter, Date, Height},
+};
+
+#[derive(Default, Debug, Encode, Decode, Allocative)]
+pub struct Metadata {
+    path: PathBuf,
+    data: MetadataData,
+}
+
+impl Deref for Metadata {
+    type Target = MetadataData;
+
+    fn deref(&self) -> &Self::Target {
+        &self.data
+    }
+}
+
+impl DerefMut for Metadata {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.data
+    }
+}
+
+impl Metadata {
+    pub fn import(path: &Path, version: u16) -> Self {
+        Self {
+            data: MetadataData::import(path, version),
+            path: path.to_owned(),
+        }
+    }
+
+    pub fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
+        if self.last_height.unwrap_or_default() < height {
+            self.last_height.replace(height);
+        }
+
+        if self.last_date.unwrap_or_default() < date {
+            self.last_date.replace(date);
+        }
+
+        self.data.export(&self.path)
+    }
+
+    pub fn reset(&mut self) {
+        let _ = self.data.reset(&self.path);
+    }
+
+    pub fn called_insert(&mut self) {
+        self.serial += 1;
+        self.len.increment();
+    }
+
+    pub fn called_remove(&mut self) {
+        self.len.decrement();
+    }
+
+    pub fn check_if_in_sync(&self, other: &Self) -> bool {
+        self.last_date == other.last_date && self.last_height == other.last_height
+    }
+
+    pub fn check_farer_or_in_sync(&self, other: &Self) -> bool {
+        self.last_date >= other.last_date && self.last_height >= other.last_height
+    }
+}
+
+#[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)]
+pub struct MetadataData {
+    version: u16,
+    pub serial: usize,
+    pub len: Counter,
+    pub last_height: Option<Height>,
+    pub last_date: Option<Date>,
+}
+
+impl MetadataData {
+    fn full_path(folder_path: &Path) -> PathBuf {
+        folder_path.join("metadata")
+    }
+
+    pub fn import(path: &Path, version: u16) -> Self {
+        let mut s = Self::_import(path, version).unwrap_or_default();
+        s.version = version;
+        s
+    }
+
+    fn _import(path: &Path, version: u16) -> color_eyre::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        let s: MetadataData = Serialization::Binary.import(&Self::full_path(path))?;
+
+        if s.version != version {
+            return Err(eyre!("Bad version"));
+        }
+
+        Ok(s)
+    }
+
+    pub fn export(&self, path: &Path) -> color_eyre::Result<()> {
+        Serialization::Binary.export(Path::new(&Self::full_path(path)), self)
+    }
+
+    pub fn reset(&mut self, path: &Path) -> color_eyre::Result<(), io::Error> {
+        self.clear();
+
+        fs::remove_file(Self::full_path(path))
+    }
+
+    fn clear(&mut self) {
+        self.serial = 0;
+        self.len.reset();
+        self.last_height = None;
+        self.last_date = None;
+    }
+}

_src/parser/databases/mod.rs

@@ -0,0 +1,175 @@
+use allocative::Allocative;
+
+mod _trait;
+mod address_index_to_address_data;
+mod address_index_to_empty_address_data;
+mod address_to_address_index;
+mod metadata;
+mod txid_to_tx_data;
+mod txout_index_to_address_index;
+mod txout_index_to_amount;
+
+use _trait::*;
+pub use address_index_to_address_data::*;
+pub use address_index_to_empty_address_data::*;
+pub use address_to_address_index::*;
+use itertools::Itertools;
+use log::info;
+use metadata::*;
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
+use snkrj::AnyDatabase;
+pub use txid_to_tx_data::*;
+pub use txout_index_to_address_index::*;
+pub use txout_index_to_amount::*;
+
+use crate::structs::{Config, Date, Height};
+
+#[derive(Allocative)]
+pub struct Databases {
+    pub address_index_to_address_data: AddressIndexToAddressData,
+    pub address_index_to_empty_address_data: AddressIndexToEmptyAddressData,
+    pub address_to_address_index: AddressToAddressIndex,
+    pub txid_to_tx_data: TxidToTxData,
+    pub txout_index_to_address_index: TxoutIndexToAddressIndex,
+    pub txout_index_to_amount: TxoutIndexToAmount,
+}
+
+impl Databases {
+    pub fn import(config: &Config) -> Self {
+        let address_index_to_address_data = AddressIndexToAddressData::init(config);
+
+        let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init(config);
+
+        let address_to_address_index = AddressToAddressIndex::init(config);
+
+        let txid_to_tx_data = TxidToTxData::init(config);
+
+        let txout_index_to_address_index = TxoutIndexToAddressIndex::init(config);
+
+        let txout_index_to_amount = TxoutIndexToAmount::init(config);
+
+        info!("Imported databases");
+
+        Self {
+            address_index_to_address_data,
+            address_index_to_empty_address_data,
+            address_to_address_index,
+            txid_to_tx_data,
+            txout_index_to_address_index,
+            txout_index_to_amount,
+        }
+    }
+
+    pub fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
+        self.txid_to_tx_data
+            .drain_to_vec()
+            .into_iter()
+            .chain(self.txout_index_to_amount.drain_to_vec())
+            .chain(self.address_to_address_index.drain_to_vec())
+            .chain(self.address_index_to_address_data.drain_to_vec())
+            .chain(self.address_index_to_empty_address_data.drain_to_vec())
+            .chain(self.txout_index_to_address_index.drain_to_vec())
+            .collect_vec()
+    }
+
+    fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
+        self.txid_to_tx_data.export_metadata(height, date)?;
+        self.txout_index_to_amount.export_metadata(height, date)?;
+        self.address_index_to_address_data
+            .export_metadata(height, date)?;
+        self.address_index_to_empty_address_data
+            .export_metadata(height, date)?;
+        self.address_to_address_index
+            .export_metadata(height, date)?;
+        self.txout_index_to_address_index
+            .export_metadata(height, date)?;
+        Ok(())
+    }
+
+    pub fn export(
+        &mut self,
+        height: Height,
+        date: Date,
+        defragment: bool,
+    ) -> color_eyre::Result<()> {
+        self.export_metadata(height, date)?;
+
+        self.drain_to_vec()
+            .into_par_iter()
+            .try_for_each(|s| AnyDatabase::boxed_export(s, defragment))?;
+
+        Ok(())
+    }
+
+    pub fn reset(&mut self, include_addresses: bool) {
+        if include_addresses {
+            let _ = self.address_index_to_address_data.reset();
+            let _ = self.address_index_to_empty_address_data.reset();
+            let _ = self.address_to_address_index.reset();
+            let _ = self.txout_index_to_address_index.reset();
+        }
+
+        let _ = self.txid_to_tx_data.reset();
+        let _ = self.txout_index_to_amount.reset();
+    }
+
+    pub fn check_if_needs_to_compute_addresses(&self, height: Height, date: Date) -> bool {
+        let check_height = |last_height: Option<Height>| {
+            last_height.map_or(true, |last_height| last_height < height)
+        };
+
+        let check_date =
+            |last_date: Option<Date>| last_date.map_or(true, |last_date| last_date < date);
+
+        let check_metadata = |metadata: &Metadata| {
+            check_height(metadata.last_height) || check_date(metadata.last_date)
+        };
+
+        // We only need to check one as we previously checked that they're all in sync
+        check_metadata(&self.address_to_address_index.metadata)
+    }
+
+    pub fn check_if_usable(
+        &self,
+        last_address_height: Option<Height>,
+        last_address_date: Option<Date>,
+    ) -> bool {
+        let are_tx_databases_in_sync = self
+            .txout_index_to_amount
+            .metadata
+            .check_if_in_sync(&self.txid_to_tx_data.metadata);
+
+        if !are_tx_databases_in_sync {
+            return false;
+        }
+
+        let are_address_databases_in_sync = self
+            .address_to_address_index
+            .metadata
+            .check_if_in_sync(&self.address_index_to_empty_address_data.metadata)
+            && self
+                .address_to_address_index
+                .metadata
+                .check_if_in_sync(&self.address_index_to_address_data.metadata)
+            && self
+                .address_to_address_index
+                .metadata
+                .check_if_in_sync(&self.txout_index_to_address_index.metadata);
+
+        if !are_address_databases_in_sync {
+            return false;
+        }
+
+        let are_address_databases_farer_or_in_sync_with_tx_database = self
+            .address_to_address_index
+            .metadata
+            .check_farer_or_in_sync(&self.txid_to_tx_data.metadata);
+
+        if !are_address_databases_farer_or_in_sync_with_tx_database {
+            return false;
+        }
+
+        last_address_height >= self.address_to_address_index.metadata.last_height
+            && last_address_date >= self.address_to_address_index.metadata.last_date
+    }
+}

_src/parser/databases/txid_to_tx_data.rs

@@ -0,0 +1,156 @@
+use std::{
+    collections::BTreeMap,
+    fs, mem,
+    path::{Path, PathBuf},
+};
+
+use allocative::Allocative;
+use brk_parser::bitcoin::Txid;
+use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
+
+use crate::structs::{Config, TxData, U8x31};
+
+use super::{AnyDatabaseGroup, Metadata};
+
+type Key = U8x31;
+type Value = TxData;
+type Database = _Database<Key, Value>;
+
+#[derive(Allocative)]
+pub struct TxidToTxData {
+    path: PathBuf,
+    pub metadata: Metadata,
+    #[allocative(skip)]
+    map: BTreeMap<u16, Database>,
+}
+
+impl TxidToTxData {
+    pub fn insert(&mut self, txid: &Txid, tx_index: Value) -> Option<Value> {
+        self.metadata.called_insert();
+
+        let txid_key = Self::txid_to_key(txid);
+
+        self.open_db(txid).insert(txid_key, tx_index)
+    }
+
+    /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed.
+    /// Though it makes it easy to use with rayon
+    pub fn get(&self, txid: &Txid) -> Option<&Value> {
+        let txid_key = Self::txid_to_key(txid);
+
+        let db_index = Self::db_index(txid);
+
+        self.map.get(&db_index).unwrap().get(&txid_key)
+    }
+
+    pub fn get_mut_from_ram(&mut self, txid: &Txid) -> Option<&mut Value> {
+        let txid_key = Self::txid_to_key(txid);
+
+        let db_index = Self::db_index(txid);
+
+        self.map.get_mut(&db_index).unwrap().get_mut_from_ram(&txid_key)
+    }
+
+    pub fn remove_later_from_disk(&mut self, txid: &Txid) {
+        self.metadata.called_remove();
+
+        let txid_key = Self::txid_to_key(txid);
+
+        self.open_db(txid).remove_later_from_disk(&txid_key);
+    }
+
+    pub fn remove_from_ram(&mut self, txid: &Txid) {
+        self.metadata.called_remove();
+
+        let txid_key = Self::txid_to_key(txid);
+
+        self.open_db(txid).remove_from_ram(&txid_key);
+    }
+
+    pub fn update(&mut self, txid: &Txid, tx_data: TxData) {
+        let txid_key = Self::txid_to_key(txid);
+
+        self.open_db(txid).update(txid_key, tx_data);
+    }
+
+    #[inline(always)]
+    pub fn open_db(&mut self, txid: &Txid) -> &mut Database {
+        let db_index = Self::db_index(txid);
+        self._open_db(db_index)
+    }
+
+    #[inline(always)]
+    fn _open_db(&mut self, db_index: u16) -> &mut Database {
+        let path = self.path.to_owned();
+        self.map.entry(db_index).or_insert_with(|| {
+            let path = path.join(db_index.to_string());
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn txid_to_key(txid: &Txid) -> U8x31 {
+        U8x31::from(&txid[1..])
+    }
+
+    fn db_index(txid: &Txid) -> u16 {
+        ((txid[0] as u16) << 5) + ((txid[1] as u16) >> 3)
+    }
+}
+
+impl AnyDatabaseGroup for TxidToTxData {
+    fn import(config: &Config) -> Self {
+        let path = config.path_databases().join("txid_to_tx_data");
+        let metadata = Metadata::import(&path, 2);
+
+        Self {
+            path,
+            metadata,
+            map: BTreeMap::default(),
+        }
+    }
+
+    fn reset_metadata(&mut self) {
+        self.metadata.reset();
+    }
+
+    fn open_all(&mut self) {
+        let folder = fs::read_dir(&self.path);
+
+        if folder.is_err() {
+            return;
+        }
+
+        folder
+            .unwrap()
+            .flat_map(|entry| {
+                entry
+                    .unwrap()
+                    .path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_owned()
+                    .parse::<u16>()
+            })
+            .for_each(|db_index| {
+                self._open_db(db_index);
+            });
+    }
+
+    fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
+        mem::take(&mut self.map)
+            .into_values()
+            .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
+            .collect_vec()
+    }
+
+    fn metadata(&mut self) -> &mut Metadata {
+        &mut self.metadata
+    }
+
+    fn path(&self) -> &Path {
+        &self.path
+    }
+}

_src/parser/databases/txout_index_to_address_index.rs

@@ -0,0 +1,148 @@
+use std::{
+    collections::BTreeMap,
+    fs, mem,
+    ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
+};
+
+use allocative::Allocative;
+use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
+
+use crate::structs::{Config, TxoutIndex};
+
+use super::{AnyDatabaseGroup, Metadata};
+
+type Key = TxoutIndex;
+type Value = u32;
+type Database = _Database<Key, Value>;
+
+#[derive(Allocative)]
+pub struct TxoutIndexToAddressIndex {
+    path: PathBuf,
+    pub metadata: Metadata,
+    #[allocative(skip)]
+    map: BTreeMap<usize, Database>,
+}
+
+impl Deref for TxoutIndexToAddressIndex {
+    type Target = BTreeMap<usize, Database>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.map
+    }
+}
+
+impl DerefMut for TxoutIndexToAddressIndex {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.map
+    }
+}
+
+const DB_MAX_SIZE: usize = 10_000_000_000;
+
+impl TxoutIndexToAddressIndex {
+    pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
+        self.metadata.called_insert();
+
+        self.open_db(&key).insert_to_ram(key, value)
+    }
+
+    pub fn remove(&mut self, key: &Key) -> Option<Value> {
+        self.metadata.called_remove();
+
+        self.open_db(key).remove(key)
+    }
+
+    /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
+    /// Though it makes it easy to use with rayon.
+    pub fn unsafe_get(&self, key: &Key) -> Option<&Value> {
+        let db_index = Self::db_index(key);
+
+        self.get(&db_index).unwrap().get(key)
+    }
+
+    pub fn open_db(&mut self, key: &Key) -> &mut Database {
+        let db_index = Self::db_index(key);
+        let path = self.path.to_owned();
+
+        self.entry(db_index).or_insert_with(|| {
+            let db_name = format!(
+                "{}..{}",
+                db_index * DB_MAX_SIZE,
+                (db_index + 1) * DB_MAX_SIZE
+            );
+
+            let path = path.join(db_name);
+
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn db_index(key: &Key) -> usize {
+        key.as_u64() as usize / DB_MAX_SIZE
+    }
+}
+
+impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
+    fn import(config: &Config) -> Self {
+        let path = config.path_databases().join("txout_index_to_address_index");
+        Self {
+            metadata: Metadata::import(&path, 1),
+            path,
+            map: BTreeMap::default(),
+        }
+    }
+
+    fn reset_metadata(&mut self) {
+        self.metadata.reset();
+    }
+
+    fn open_all(&mut self) {
+        let folder = fs::read_dir(&self.path);
+
+        if folder.is_err() {
+            return;
+        }
+
+        folder
+            .unwrap()
+            .map(|entry| {
+                entry
+                    .unwrap()
+                    .path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_owned()
+            })
+            .filter(|file_name| file_name.contains(".."))
+            .for_each(|path| {
+                self.open_db(
+                    &path
+                        .split("..")
+                        .next()
+                        .unwrap()
+                        .parse::<u64>()
+                        .unwrap()
+                        .into(),
+                );
+            });
+    }
+
+    fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
+        mem::take(&mut self.map)
+            .into_values()
+            .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
+            .collect_vec()
+    }
+
+    fn metadata(&mut self) -> &mut Metadata {
+        &mut self.metadata
+    }
+
+    fn path(&self) -> &Path {
+        &self.path
+    }
+}

_src/parser/databases/txout_index_to_amount.rs

@@ -0,0 +1,148 @@
+use std::{
+    collections::BTreeMap,
+    fs, mem,
+    ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
+};
+
+use allocative::Allocative;
+use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
+
+use crate::structs::{Amount, Config, TxoutIndex};
+
+use super::{AnyDatabaseGroup, Metadata};
+
+type Key = TxoutIndex;
+type Value = Amount;
+type Database = _Database<Key, Value>;
+
+#[derive(Allocative)]
+pub struct TxoutIndexToAmount {
+    path: PathBuf,
+    pub metadata: Metadata,
+    #[allocative(skip)]
+    map: BTreeMap<usize, Database>,
+}
+
+impl Deref for TxoutIndexToAmount {
+    type Target = BTreeMap<usize, Database>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.map
+    }
+}
+
+impl DerefMut for TxoutIndexToAmount {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.map
+    }
+}
+
+const DB_MAX_SIZE: usize = 10_000_000_000;
+
+impl TxoutIndexToAmount {
+    pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
+        self.metadata.called_insert();
+
+        self.open_db(&key).insert_to_ram(key, value)
+    }
+
+    pub fn remove(&mut self, key: &Key) -> Option<Value> {
+        self.metadata.called_remove();
+
+        self.open_db(key).remove(key)
+    }
+
+    /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
+    /// Though it makes it easy to use with rayon.
+    pub fn unsafe_get(&self, key: &Key) -> Option<&Value> {
+        let db_index = Self::db_index(key);
+
+        self.get(&db_index).unwrap().get(key)
+    }
+
+    pub fn open_db(&mut self, key: &Key) -> &mut Database {
+        let db_index = Self::db_index(key);
+        let path = self.path.to_owned();
+
+        self.entry(db_index).or_insert_with(|| {
+            let db_name = format!(
+                "{}..{}",
+                db_index * DB_MAX_SIZE,
+                (db_index + 1) * DB_MAX_SIZE
+            );
+
+            let path = path.join(db_name);
+
+            Database::open(path).unwrap()
+        })
+    }
+
+    fn db_index(key: &Key) -> usize {
+        key.as_u64() as usize / DB_MAX_SIZE
+    }
+}
+
+impl AnyDatabaseGroup for TxoutIndexToAmount {
+    fn import(config: &Config) -> Self {
+        let path = config.path_databases().join("txout_index_to_amount");
+        Self {
+            metadata: Metadata::import(&path, 1),
+            path,
+            map: BTreeMap::default(),
+        }
+    }
+
+    fn reset_metadata(&mut self) {
+        self.metadata.reset();
+    }
+
+    fn open_all(&mut self) {
+        let folder = fs::read_dir(&self.path);
+
+        if folder.is_err() {
+            return;
+        }
+
+        folder
+            .unwrap()
+            .map(|entry| {
+                entry
+                    .unwrap()
+                    .path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_owned()
+            })
+            .filter(|file_name| file_name.contains(".."))
+            .for_each(|path| {
+                self.open_db(
+                    &path
+                        .split("..")
+                        .next()
+                        .unwrap()
+                        .parse::<u64>()
+                        .unwrap()
+                        .into(),
+                );
+            });
+    }
+
+    fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
+        mem::take(&mut self.map)
+            .into_values()
+            .map(|db| Box::new(db) as Box<dyn AnyDatabase + Send>)
+            .collect_vec()
+    }
+
+    fn metadata(&mut self) -> &mut Metadata {
+        &mut self.metadata
+    }
+
+    fn path(&self) -> &Path {
+        &self.path
+    }
+}