diff --git a/README.md b/README.md index b622cde76..5304006a2 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Here's an example ./run.sh --datadir=$HOME/Developer/bitcoin --rpcuser=satoshi --rpcpassword=nakamoto ``` -In a new terminal, go to the `server`'s folder of the repository +In a **new** terminal, go to the `server`'s folder of the repository ```bash cd ???/kibo/server diff --git a/parser/src/databases/_database.rs b/parser/src/databases/_database.rs index af9fa270d..abc695f56 100644 --- a/parser/src/databases/_database.rs +++ b/parser/src/databases/_database.rs @@ -2,12 +2,11 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, fs, mem, + path::PathBuf, }; use allocative::Allocative; -use derive_deref::{Deref, DerefMut}; -use itertools::Itertools; // https://docs.rs/sanakirja/latest/sanakirja/index.html // https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ // @@ -17,11 +16,9 @@ use itertools::Itertools; // Possible compression: https://pijul.org/posts/sanakirja-zstd/ use sanakirja::{ btree::{self, page, Db_, Iter}, - direct_repr, Commit, Env, Error, MutTxn, RootDb, Storable, UnsizedStorable, + Commit, Env, Error, MutTxn, RootDb, Storable, }; -use crate::io::OUTPUTS_FOLDER_PATH; - #[derive(Allocative)] #[allocative(bound = "Key: Allocative, Value: Allocative")] /// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way. @@ -32,8 +29,7 @@ where { pub cached_puts: BTreeMap, pub cached_dels: BTreeSet, - folder: String, - file: String, + path: PathBuf, #[allocative(skip)] db: Db_>, #[allocative(skip)] @@ -48,12 +44,8 @@ where Key: Ord + Clone + Debug + Storable, Value: Storable + PartialEq, { - pub fn open(folder: &str, file: &str) -> color_eyre::Result { - let path = databases_folder_path(folder); - fs::create_dir_all(&path)?; - - let path = format!("{path}/{file}"); - let env = unsafe { Env::new_nolock(path, PAGE_SIZE, 1).unwrap() }; + pub fn open(path: PathBuf) -> color_eyre::Result { + let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? }; let mut txn = Env::mut_txn_begin(env)?; @@ -62,8 +54,7 @@ where .unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() }); Ok(Self { - folder: folder.to_owned(), - file: file.to_owned(), + path, cached_puts: BTreeMap::default(), cached_dels: BTreeSet::default(), db, @@ -93,17 +84,6 @@ where self.db_get(key) } - fn destroy(self) { - let path = self.path(); - - drop(self); - - fs::remove_file(&path).unwrap_or_else(|_| { - dbg!(path); - panic!("Error"); - }); - } - pub fn db_get(&self, key: &Key) -> Option<&Value> { let option = btree::get(&self.txn, &self.db, key, None).unwrap(); @@ -135,46 +115,39 @@ where }) } - #[inline(always)] + #[inline] pub fn db_remove(&mut self, key: &Key) { self.cached_dels.insert(key.clone()); } + #[inline] pub fn update(&mut self, key: Key, value: Value) -> Option { self.cached_dels.insert(key.clone()); - self.cached_puts.insert(key, value) } - fn len(&self) -> usize { - self.iter().try_len().unwrap_or_else(|e| e.0) + #[inline] + pub fn is_empty(&self) -> bool { + self.iter().next().is_none() } - fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[inline(always)] + #[inline] pub fn remove_from_puts(&mut self, key: &Key) -> Option { self.cached_puts.remove(key) } - #[inline(always)] + #[inline] pub fn insert(&mut self, key: Key, value: Value) -> Option { self.cached_dels.remove(&key); self.unsafe_insert(key, value) } - #[inline(always)] + #[inline] pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { self.cached_puts.insert(key, value) } - fn path(&self) -> String { - format!("{}/{}", databases_folder_path(&self.folder), self.file) - } - fn db_multi_put(&mut self, tree: BTreeMap) -> Result<(), Error> { tree.into_iter() .try_for_each(|(key, value)| -> Result<(), Error> { @@ -197,6 +170,7 @@ pub trait AnyDatabase { #[allow(unused)] fn defragment(self); fn boxed_defragment(self: Box); + fn destroy(self); } impl AnyDatabase for Database @@ -231,48 +205,28 @@ where fn boxed_defragment(self: Box) { let btree = self.iter_collect(); - let folder = self.folder.to_owned(); - let file = self.file.to_owned(); + let path = self.path.to_owned(); self.destroy(); - let mut s = Self::open(&folder, &file).unwrap(); + let mut db = Self::open(path).unwrap(); - if !s.is_empty() { + if !db.is_empty() { panic!() } - s.cached_puts = btree; - s.export().unwrap(); + db.cached_puts = btree; + db.export().unwrap(); + } + + fn destroy(self) { + let path = self.path.to_owned(); + + drop(self); + + fs::remove_file(&path).unwrap_or_else(|_| { + dbg!(path); + panic!("Error"); + }); } } - -#[derive( - Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative, -)] -pub struct U8x19([u8; 19]); -direct_repr!(U8x19); -impl From<&[u8]> for U8x19 { - fn from(slice: &[u8]) -> Self { - let mut arr = Self::default(); - arr.copy_from_slice(slice); - arr - } -} - -#[derive( - Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative, -)] -pub struct U8x31([u8; 31]); -direct_repr!(U8x31); -impl From<&[u8]> for U8x31 { - fn from(slice: &[u8]) -> Self { - let mut arr = Self::default(); - arr.copy_from_slice(slice); - arr - } -} - -pub fn databases_folder_path(folder: &str) -> String { - format!("{OUTPUTS_FOLDER_PATH}/databases/{folder}") -} diff --git a/parser/src/databases/_trait.rs b/parser/src/databases/_trait.rs index 41c78aff8..acf040d0b 100644 --- a/parser/src/databases/_trait.rs +++ b/parser/src/databases/_trait.rs @@ -1,16 +1,23 @@ -use std::{fs, io}; +use std::{fs, io, path::PathBuf}; use crate::{ + io::OUTPUTS_FOLDER_PATH, structs::{Date, Height}, utils::log, }; -use super::{databases_folder_path, AnyDatabase}; +use super::AnyDatabase; pub trait AnyDatabaseGroup where Self: Sized, { + fn init() -> Self { + let s = Self::import(); + s.create_dir_all().unwrap(); + s + } + fn import() -> Self; fn folder<'a>() -> &'a str; @@ -19,22 +26,27 @@ where fn open_all(&mut self); fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()>; - // fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()>; - // fn defragment(&mut self); + + fn create_dir_all(&self) -> color_eyre::Result<(), io::Error>; + + fn remove_dir_all(&self) -> color_eyre::Result<(), io::Error> { + fs::remove_dir_all(Self::root()) + } fn reset(&mut self) -> color_eyre::Result<(), io::Error> { log(&format!("Reset {}", Self::folder())); self.reset_metadata(); - - fs::remove_dir_all(Self::full_path())?; + self.remove_dir_all()?; + self.create_dir_all()?; Ok(()) } - fn full_path() -> String { - databases_folder_path(Self::folder()) - } - fn reset_metadata(&mut self); + + fn root() -> PathBuf { + let folder = Self::folder(); + PathBuf::from(format!("{OUTPUTS_FOLDER_PATH}/databases/{folder}")) + } } diff --git a/parser/src/databases/address_index_to_address_data.rs b/parser/src/databases/address_index_to_address_data.rs index 791fa9eb6..e6ed7fc80 100644 --- a/parser/src/databases/address_index_to_address_data.rs +++ b/parser/src/databases/address_index_to_address_data.rs @@ -80,7 +80,9 @@ impl AddressIndexToAddressData { (db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE ); - Database::open(Self::folder(), &db_name).unwrap() + let path = Self::root().join(db_name); + + Database::open(path).unwrap() }) } @@ -113,12 +115,16 @@ impl AddressIndexToAddressData { impl AnyDatabaseGroup for AddressIndexToAddressData { fn import() -> Self { Self { - metadata: Metadata::import(&Self::full_path(), 1), + metadata: Metadata::import(Self::root(), 1), map: BTreeMap::default(), } } + fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> { + fs::create_dir_all(Self::root()) + } + fn reset_metadata(&mut self) { self.metadata.reset(); } @@ -128,7 +134,7 @@ impl AnyDatabaseGroup for AddressIndexToAddressData { } fn open_all(&mut self) { - let path = Self::full_path(); + let path = Self::root(); let folder = fs::read_dir(path); diff --git a/parser/src/databases/address_index_to_empty_address_data.rs b/parser/src/databases/address_index_to_empty_address_data.rs index 337a568f1..cdb0127fc 100644 --- a/parser/src/databases/address_index_to_empty_address_data.rs +++ b/parser/src/databases/address_index_to_empty_address_data.rs @@ -80,7 +80,9 @@ impl AddressIndexToEmptyAddressData { (db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE ); - Database::open(Self::folder(), &db_name).unwrap() + let path = Self::root().join(db_name); + + Database::open(path).unwrap() }) } @@ -92,12 +94,16 @@ impl AddressIndexToEmptyAddressData { impl AnyDatabaseGroup for AddressIndexToEmptyAddressData { fn import() -> Self { Self { - metadata: Metadata::import(&Self::full_path(), 1), + metadata: Metadata::import(Self::root(), 1), map: BTreeMap::default(), } } + fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> { + fs::create_dir_all(Self::root()) + } + fn reset_metadata(&mut self) { self.metadata.reset(); } @@ -107,7 +113,7 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData { } fn open_all(&mut self) { - let path = Self::full_path(); + let path = Self::root(); let folder = fs::read_dir(path); @@ -143,22 +149,4 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData { fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> { self.metadata.export(height, date) } - - // fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> { - // self.drain_to_vec() - // .into_par_iter() - // .try_for_each(AnyDatabase::boxed_export)?; - - // self.metadata.export(height, date)?; - - // Ok(()) - // } - - // fn defragment(&mut self) { - // self.open_all(); - - // self.drain_to_vec() - // .into_par_iter() - // .for_each(AnyDatabase::boxed_defragment); - // } } diff --git a/parser/src/databases/address_to_address_index.rs b/parser/src/databases/address_to_address_index.rs index d3a559c67..34b7e8d5b 100644 --- a/parser/src/databases/address_to_address_index.rs +++ b/parser/src/databases/address_to_address_index.rs @@ -1,13 +1,15 @@ -use std::{collections::BTreeMap, fs, mem}; +use std::{ + collections::BTreeMap, + fs, mem, + path::{Path, PathBuf}, +}; use allocative::Allocative; use itertools::Itertools; -use crate::structs::{Address, Date, Height}; +use crate::structs::{Address, Date, Height, U8x19, U8x31}; -use super::{ - databases_folder_path, AnyDatabase, AnyDatabaseGroup, Database, Metadata, U8x19, U8x31, -}; +use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata}; type Value = u32; type U8x19Database = Database; @@ -44,20 +46,6 @@ pub struct AddressToAddressIndex { } impl AddressToAddressIndex { - // pub fn safe_get(&mut self, address: &Address) -> Option<&Value> { - // match address { - // Address::Empty(key) => self.open_empty().get(key), - // Address::Unknown(key) => self.open_unknown().get(key), - // Address::MultiSig(key) => self.open_multisig().get(key), - // Address::P2PK((prefix, rest)) => self.open_p2pk(*prefix).get(rest), - // Address::P2PKH((prefix, rest)) => self.open_p2pkh(*prefix).get(rest), - // Address::P2SH((prefix, rest)) => self.open_p2sh(*prefix).get(rest), - // Address::P2WPKH((prefix, rest)) => self.open_p2wpkh(*prefix).get(rest), - // Address::P2WSH((prefix, rest)) => self.open_p2wsh(*prefix).get(rest), - // Address::P2TR((prefix, rest)) => self.open_p2tr(*prefix).get(rest), - // } - // } - pub fn open_db(&mut self, address: &Address) { match address { Address::Empty(_) => { @@ -148,9 +136,7 @@ impl AddressToAddressIndex { } } - fn path_to_group_prefixes(path: &str) -> Vec { - let path = databases_folder_path(path); - + fn path_to_group_prefixes(path: &Path) -> Vec { let folder = fs::read_dir(path); if folder.is_err() { @@ -174,15 +160,16 @@ impl AddressToAddressIndex { .collect_vec() } - fn path_p2pk() -> String { - format!("{}/{}", Self::folder(), "p2pk") + fn path_p2pk() -> PathBuf { + Self::root().join("p2pk") } pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase { let path = Self::path_p2pk(); - self.p2pk - .entry(prefix) - .or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap()) + self.p2pk.entry(prefix).or_insert_with(|| { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }) } fn open_all_p2pk(&mut self) { @@ -190,21 +177,24 @@ impl AddressToAddressIndex { Self::path_to_group_prefixes(&path) .into_iter() .for_each(|prefix| { - self.p2pk - .insert(prefix, Database::open(&path, &prefix.to_string()).unwrap()); + self.p2pk.insert(prefix, { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }); }); } - fn path_p2pkh() -> String { - format!("{}/{}", Self::folder(), "p2pkh") + fn path_p2pkh() -> PathBuf { + Self::root().join("p2pkh") } pub fn open_p2pkh(&mut self, prefix: u16) -> &mut P2PKHDatabase { let path = Self::path_p2pkh(); - self.p2pkh - .entry(prefix) - .or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap()) + self.p2pkh.entry(prefix).or_insert_with(|| { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }) } fn open_all_p2pkh(&mut self) { @@ -212,21 +202,24 @@ impl AddressToAddressIndex { Self::path_to_group_prefixes(&path) .into_iter() .for_each(|prefix| { - self.p2pkh - .insert(prefix, Database::open(&path, &prefix.to_string()).unwrap()); + self.p2pkh.insert(prefix, { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }); }); } - fn path_p2sh() -> String { - format!("{}/{}", Self::folder(), "p2sh") + fn path_p2sh() -> PathBuf { + Self::root().join("p2sh") } pub fn open_p2sh(&mut self, prefix: u16) -> &mut P2SHDatabase { let path = Self::path_p2sh(); - self.p2sh - .entry(prefix) - .or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap()) + self.p2sh.entry(prefix).or_insert_with(|| { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }) } fn open_all_p2sh(&mut self) { @@ -234,21 +227,24 @@ impl AddressToAddressIndex { Self::path_to_group_prefixes(&path) .into_iter() .for_each(|prefix| { - self.p2sh - .insert(prefix, Database::open(&path, &prefix.to_string()).unwrap()); + self.p2sh.insert(prefix, { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }); }); } - fn path_p2wpkh() -> String { - format!("{}/{}", Self::folder(), "p2wpkh") + fn path_p2wpkh() -> PathBuf { + Self::root().join("p2wpkh") } pub fn open_p2wpkh(&mut self, prefix: u16) -> &mut P2WPKHDatabase { let path = Self::path_p2wpkh(); - self.p2wpkh - .entry(prefix) - .or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap()) + self.p2wpkh.entry(prefix).or_insert_with(|| { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }) } fn open_all_p2wpkh(&mut self) { @@ -256,21 +252,24 @@ impl AddressToAddressIndex { Self::path_to_group_prefixes(&path) .into_iter() .for_each(|prefix| { - self.p2wpkh - .insert(prefix, Database::open(&path, &prefix.to_string()).unwrap()); + self.p2wpkh.insert(prefix, { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }); }); } - fn path_p2wsh() -> String { - format!("{}/{}", Self::folder(), "p2wsh") + fn path_p2wsh() -> PathBuf { + Self::root().join("p2wsh") } pub fn open_p2wsh(&mut self, prefix: u16) -> &mut P2WSHDatabase { let path = Self::path_p2wsh(); - self.p2wsh - .entry(prefix) - .or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap()) + self.p2wsh.entry(prefix).or_insert_with(|| { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }) } fn open_all_p2wsh(&mut self) { @@ -278,21 +277,24 @@ impl AddressToAddressIndex { Self::path_to_group_prefixes(&path) .into_iter() .for_each(|prefix| { - self.p2wsh - .insert(prefix, Database::open(&path, &prefix.to_string()).unwrap()); + self.p2wsh.insert(prefix, { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }); }); } - fn path_p2tr() -> String { - format!("{}/{}", Self::folder(), "p2tr") + fn path_p2tr() -> PathBuf { + Self::root().join("p2tr") } pub fn open_p2tr(&mut self, prefix: u16) -> &mut P2TRDatabase { let path = Self::path_p2tr(); - self.p2tr - .entry(prefix) - .or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap()) + self.p2tr.entry(prefix).or_insert_with(|| { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }) } fn open_all_p2tr(&mut self) { @@ -300,41 +302,43 @@ impl AddressToAddressIndex { Self::path_to_group_prefixes(&path) .into_iter() .for_each(|prefix| { - self.p2tr - .insert(prefix, Database::open(&path, &prefix.to_string()).unwrap()); + self.p2tr.insert(prefix, { + let path = path.join(prefix.to_string()); + Database::open(path).unwrap() + }); }); } pub fn open_unknown(&mut self) -> &mut UnknownDatabase { self.unknown - .get_or_insert_with(|| Database::open(Self::folder(), "unknown").unwrap()) + .get_or_insert_with(|| Database::open(Self::root().join("unknown")).unwrap()) } pub fn open_op_return(&mut self) -> &mut UnknownDatabase { self.op_return - .get_or_insert_with(|| Database::open(Self::folder(), "op_return").unwrap()) + .get_or_insert_with(|| Database::open(Self::root().join("op_return")).unwrap()) } pub fn open_push_only(&mut self) -> &mut UnknownDatabase { self.push_only - .get_or_insert_with(|| Database::open(Self::folder(), "push_only").unwrap()) + .get_or_insert_with(|| Database::open(Self::root().join("push_only")).unwrap()) } pub fn open_empty(&mut self) -> &mut UnknownDatabase { self.empty - .get_or_insert_with(|| Database::open(Self::folder(), "empty").unwrap()) + .get_or_insert_with(|| Database::open(Self::root().join("empty")).unwrap()) } pub fn open_multisig(&mut self) -> &mut MultisigDatabase { self.multisig - .get_or_insert_with(|| Database::open(Self::folder(), "multisig").unwrap()) + .get_or_insert_with(|| Database::open(Self::root().join("multisig")).unwrap()) } } impl AnyDatabaseGroup for AddressToAddressIndex { fn import() -> Self { Self { - metadata: Metadata::import(&Self::full_path(), 1), + metadata: Metadata::import(Self::root(), 1), p2pk: BTreeMap::default(), p2pkh: BTreeMap::default(), @@ -350,6 +354,15 @@ impl AnyDatabaseGroup for AddressToAddressIndex { } } + fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> { + fs::create_dir_all(Self::path_p2pk()).unwrap(); + fs::create_dir_all(Self::path_p2pkh()).unwrap(); + fs::create_dir_all(Self::path_p2sh()).unwrap(); + fs::create_dir_all(Self::path_p2wpkh()).unwrap(); + fs::create_dir_all(Self::path_p2wsh()).unwrap(); + fs::create_dir_all(Self::path_p2tr()) + } + fn reset_metadata(&mut self) { self.metadata.reset() } diff --git a/parser/src/databases/metadata.rs b/parser/src/databases/metadata.rs index e43b62d45..0ca6781dc 100644 --- a/parser/src/databases/metadata.rs +++ b/parser/src/databases/metadata.rs @@ -6,7 +6,7 @@ use std::{ fmt::Debug, fs, io, ops::{Deref, DerefMut}, - path::Path, + path::{Path, PathBuf}, }; use crate::{ @@ -16,7 +16,7 @@ use crate::{ #[derive(Default, Debug, Encode, Decode, Allocative)] pub struct Metadata { - path: String, + path: PathBuf, data: MetadataData, } @@ -35,10 +35,10 @@ impl DerefMut for Metadata { } impl Metadata { - pub fn import(path: &str, version: u16) -> Self { + pub fn import(path: PathBuf, version: u16) -> Self { Self { - path: path.to_owned(), - data: MetadataData::import(path, version), + data: MetadataData::import(&path, version), + path, } } @@ -86,25 +86,20 @@ pub struct MetadataData { } impl MetadataData { - fn name<'a>() -> &'a str { - "metadata" + fn full_path(folder_path: &Path) -> PathBuf { + folder_path.join("metadata") } - fn full_path(folder_path: &str) -> String { - let name = Self::name(); - format!("{folder_path}/{name}") - } - - pub fn import(path: &str, version: u16) -> Self { + pub fn import(path: &Path, version: u16) -> Self { let mut s = Self::_import(path, version).unwrap_or_default(); s.version = version; s } - fn _import(path: &str, version: u16) -> color_eyre::Result { + fn _import(path: &Path, version: u16) -> color_eyre::Result { fs::create_dir_all(path)?; - let s: MetadataData = Serialization::Binary.import(Path::new(&Self::full_path(path)))?; + let s: MetadataData = Serialization::Binary.import(path)?; if s.version != version { return Err(eyre!("Bad version")); @@ -113,11 +108,11 @@ impl MetadataData { Ok(s) } - pub fn export(&self, path: &str) -> color_eyre::Result<()> { + pub fn export(&self, path: &Path) -> color_eyre::Result<()> { Serialization::Binary.export(Path::new(&Self::full_path(path)), self) } - pub fn reset(&mut self, path: &str) -> color_eyre::Result<(), io::Error> { + pub fn reset(&mut self, path: &Path) -> color_eyre::Result<(), io::Error> { self.clear(); fs::remove_file(Self::full_path(path)) diff --git a/parser/src/databases/mod.rs b/parser/src/databases/mod.rs index 08f887dda..9ff6bc752 100644 --- a/parser/src/databases/mod.rs +++ b/parser/src/databases/mod.rs @@ -43,17 +43,17 @@ pub struct Databases { impl Databases { pub fn import() -> Self { - let address_index_to_address_data = AddressIndexToAddressData::import(); + let address_index_to_address_data = AddressIndexToAddressData::init(); - let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::import(); + let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init(); - let address_to_address_index = AddressToAddressIndex::import(); + let address_to_address_index = AddressToAddressIndex::init(); - let txid_to_tx_data = TxidToTxData::import(); + let txid_to_tx_data = TxidToTxData::init(); - let txout_index_to_address_index = TxoutIndexToAddressIndex::import(); + let txout_index_to_address_index = TxoutIndexToAddressIndex::init(); - let txout_index_to_amount = TxoutIndexToAmount::import(); + let txout_index_to_amount = TxoutIndexToAmount::init(); Self { address_index_to_address_data, diff --git a/parser/src/databases/txid_to_tx_data.rs b/parser/src/databases/txid_to_tx_data.rs index 37d38b024..3e076cd0b 100644 --- a/parser/src/databases/txid_to_tx_data.rs +++ b/parser/src/databases/txid_to_tx_data.rs @@ -8,9 +8,9 @@ use allocative::Allocative; use biter::bitcoin::Txid; use itertools::Itertools; -use crate::structs::{Date, Height, TxData}; +use crate::structs::{Date, Height, TxData, U8x31}; -use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, U8x31}; +use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata}; type Key = U8x31; type Value = TxData; @@ -109,8 +109,10 @@ impl TxidToTxData { #[inline(always)] pub fn _open_db(&mut self, db_index: u16) -> &mut Database { - self.entry(db_index) - .or_insert_with(|| Database::open(Self::folder(), &db_index.to_string()).unwrap()) + self.entry(db_index).or_insert_with(|| { + let path = Self::root().join(db_index.to_string()); + Database::open(path).unwrap() + }) } fn txid_to_key(txid: &Txid) -> U8x31 { @@ -125,12 +127,16 @@ impl TxidToTxData { impl AnyDatabaseGroup for TxidToTxData { fn import() -> Self { Self { - metadata: Metadata::import(&Self::full_path(), 2), + metadata: Metadata::import(Self::root(), 2), map: BTreeMap::default(), } } + fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> { + fs::create_dir_all(Self::root()) + } + fn reset_metadata(&mut self) { self.metadata.reset(); } @@ -140,7 +146,7 @@ impl AnyDatabaseGroup for TxidToTxData { } fn open_all(&mut self) { - let path = Self::full_path(); + let path = Self::root(); let folder = fs::read_dir(path); diff --git a/parser/src/databases/txout_index_to_address_index.rs b/parser/src/databases/txout_index_to_address_index.rs index 495770035..79baeac10 100644 --- a/parser/src/databases/txout_index_to_address_index.rs +++ b/parser/src/databases/txout_index_to_address_index.rs @@ -77,7 +77,9 @@ impl TxoutIndexToAddressIndex { (db_index + 1) * DB_MAX_SIZE ); - Database::open(Self::folder(), &db_name).unwrap() + let path = Self::root().join(db_name); + + Database::open(path).unwrap() }) } @@ -89,21 +91,15 @@ impl TxoutIndexToAddressIndex { impl AnyDatabaseGroup for TxoutIndexToAddressIndex { fn import() -> Self { Self { - metadata: Metadata::import(&Self::full_path(), 1), + metadata: Metadata::import(Self::root(), 1), map: BTreeMap::default(), } } - // fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> { - // mem::take(&mut self.map) - // .into_par_iter() - // .try_for_each(|(_, db)| db.export())?; - - // self.metadata.export(height, date)?; - - // Ok(()) - // } + fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> { + fs::create_dir_all(Self::root()) + } fn reset_metadata(&mut self) { self.metadata.reset(); @@ -114,7 +110,7 @@ impl AnyDatabaseGroup for TxoutIndexToAddressIndex { } fn open_all(&mut self) { - let path = Self::full_path(); + let path = Self::root(); let folder = fs::read_dir(path); diff --git a/parser/src/databases/txout_index_to_amount.rs b/parser/src/databases/txout_index_to_amount.rs index 6344edfd4..6774dfdb4 100644 --- a/parser/src/databases/txout_index_to_amount.rs +++ b/parser/src/databases/txout_index_to_amount.rs @@ -77,7 +77,9 @@ impl TxoutIndexToAmount { (db_index + 1) * DB_MAX_SIZE ); - Database::open(Self::folder(), &db_name).unwrap() + let path = Self::root().join(db_name); + + Database::open(path).unwrap() }) } @@ -89,12 +91,16 @@ impl TxoutIndexToAmount { impl AnyDatabaseGroup for TxoutIndexToAmount { fn import() -> Self { Self { - metadata: Metadata::import(&Self::full_path(), 1), + metadata: Metadata::import(Self::root(), 1), map: BTreeMap::default(), } } + fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> { + fs::create_dir_all(Self::root()) + } + fn reset_metadata(&mut self) { self.metadata.reset(); } @@ -104,7 +110,7 @@ impl AnyDatabaseGroup for TxoutIndexToAmount { } fn open_all(&mut self) { - let path = Self::full_path(); + let path = Self::root(); let folder = fs::read_dir(path); diff --git a/parser/src/structs/address.rs b/parser/src/structs/address.rs index 12320dda5..7fc0255d5 100644 --- a/parser/src/structs/address.rs +++ b/parser/src/structs/address.rs @@ -1,9 +1,7 @@ use bitcoin_hashes::{hash160, Hash}; use biter::bitcoin::TxOut; -use crate::databases::{U8x19, U8x31}; - -use super::{AddressType, Counter}; +use super::{AddressType, Counter, U8x19, U8x31}; #[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq)] pub enum Address { diff --git a/parser/src/structs/array.rs b/parser/src/structs/array.rs new file mode 100644 index 000000000..dbfb67592 --- /dev/null +++ b/parser/src/structs/array.rs @@ -0,0 +1,31 @@ +use std::fmt::Debug; + +use allocative::Allocative; +use derive_deref::{Deref, DerefMut}; +use sanakirja::{direct_repr, Storable, UnsizedStorable}; + +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative, +)] +pub struct U8x19([u8; 19]); +direct_repr!(U8x19); +impl From<&[u8]> for U8x19 { + fn from(slice: &[u8]) -> Self { + let mut arr = Self::default(); + arr.copy_from_slice(slice); + arr + } +} + +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative, +)] +pub struct U8x31([u8; 31]); +direct_repr!(U8x31); +impl From<&[u8]> for U8x31 { + fn from(slice: &[u8]) -> Self { + let mut arr = Self::default(); + arr.copy_from_slice(slice); + arr + } +} diff --git a/parser/src/structs/mod.rs b/parser/src/structs/mod.rs index 54b0638f7..1278f5844 100644 --- a/parser/src/structs/mod.rs +++ b/parser/src/structs/mod.rs @@ -7,6 +7,7 @@ mod address_split; mod address_type; mod amount; mod any_map; +mod array; mod bi_map; mod block_data; mod block_path; @@ -44,6 +45,7 @@ pub use address_split::*; pub use address_type::*; pub use amount::*; pub use any_map::*; +pub use array::*; pub use bi_map::*; pub use block_data::*; pub use block_path::*;