Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-14 08:43:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: databases: small changes

Browse Source
This commit is contained in:
k
2024-11-04 11:09:54 +01:00
parent 2142847de3
commit 781810ed9c
14 changed files with 236 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -141,7 +141,7 @@ Here's an example
./run.sh --datadir=$HOME/Developer/bitcoin --rpcuser=satoshi --rpcpassword=nakamoto
```
In a new terminal, go to the `server`'s folder of the repository
In a **new** terminal, go to the `server`'s folder of the repository
```bash
cd ???/kibo/server
parser/src/databases/_database.rs
+31 -77
View File
@@ -2,12 +2,11 @@ use std::{
collections::{BTreeMap, BTreeSet},
fmt::Debug,
fs, mem,
path::PathBuf,
};
use allocative::Allocative;
use derive_deref::{Deref, DerefMut};
use itertools::Itertools;
// https://docs.rs/sanakirja/latest/sanakirja/index.html
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
//
@@ -17,11 +16,9 @@ use itertools::Itertools;
// Possible compression: https://pijul.org/posts/sanakirja-zstd/
use sanakirja::{
btree::{self, page, Db_, Iter},
direct_repr, Commit, Env, Error, MutTxn, RootDb, Storable, UnsizedStorable,
Commit, Env, Error, MutTxn, RootDb, Storable,
};
use crate::io::OUTPUTS_FOLDER_PATH;
#[derive(Allocative)]
#[allocative(bound = "Key: Allocative, Value: Allocative")]
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way.
@@ -32,8 +29,7 @@ where
{
pub cached_puts: BTreeMap<Key, Value>,
pub cached_dels: BTreeSet<Key>,
folder: String,
file: String,
path: PathBuf,
#[allocative(skip)]
db: Db_<Key, Value, page::Page<Key, Value>>,
#[allocative(skip)]
@@ -48,12 +44,8 @@ where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq,
{
pub fn open(folder: &str, file: &str) -> color_eyre::Result<Self> {
let path = databases_folder_path(folder);
fs::create_dir_all(&path)?;
let path = format!("{path}/{file}");
let env = unsafe { Env::new_nolock(path, PAGE_SIZE, 1).unwrap() };
pub fn open(path: PathBuf) -> color_eyre::Result<Self> {
let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? };
let mut txn = Env::mut_txn_begin(env)?;
@@ -62,8 +54,7 @@ where
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
Ok(Self {
folder: folder.to_owned(),
file: file.to_owned(),
path,
cached_puts: BTreeMap::default(),
cached_dels: BTreeSet::default(),
db,
@@ -93,17 +84,6 @@ where
self.db_get(key)
}
fn destroy(self) {
let path = self.path();
drop(self);
fs::remove_file(&path).unwrap_or_else(|_| {
dbg!(path);
panic!("Error");
});
}
pub fn db_get(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
@@ -135,46 +115,39 @@ where
})
}
#[inline(always)]
#[inline]
pub fn db_remove(&mut self, key: &Key) {
self.cached_dels.insert(key.clone());
}
#[inline]
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.insert(key.clone());
self.cached_puts.insert(key, value)
}
fn len(&self) -> usize {
self.iter().try_len().unwrap_or_else(|e| e.0)
#[inline]
pub fn is_empty(&self) -> bool {
self.iter().next().is_none()
}
fn is_empty(&self) -> bool {
self.len() == 0
}
#[inline(always)]
#[inline]
pub fn remove_from_puts(&mut self, key: &Key) -> Option<Value> {
self.cached_puts.remove(key)
}
#[inline(always)]
#[inline]
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.remove(&key);
self.unsafe_insert(key, value)
}
#[inline(always)]
#[inline]
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_puts.insert(key, value)
}
fn path(&self) -> String {
format!("{}/{}", databases_folder_path(&self.folder), self.file)
}
fn db_multi_put(&mut self, tree: BTreeMap<Key, Value>) -> Result<(), Error> {
tree.into_iter()
.try_for_each(|(key, value)| -> Result<(), Error> {
@@ -197,6 +170,7 @@ pub trait AnyDatabase {
#[allow(unused)]
fn defragment(self);
fn boxed_defragment(self: Box<Self>);
fn destroy(self);
}
impl<Key, Value> AnyDatabase for Database<Key, Value>
@@ -231,48 +205,28 @@ where
fn boxed_defragment(self: Box<Self>) {
let btree = self.iter_collect();
let folder = self.folder.to_owned();
let file = self.file.to_owned();
let path = self.path.to_owned();
self.destroy();
let mut s = Self::open(&folder, &file).unwrap();
let mut db = Self::open(path).unwrap();
if !s.is_empty() {
if !db.is_empty() {
panic!()
}
s.cached_puts = btree;
s.export().unwrap();
db.cached_puts = btree;
db.export().unwrap();
}
fn destroy(self) {
let path = self.path.to_owned();
drop(self);
fs::remove_file(&path).unwrap_or_else(|_| {
dbg!(path);
panic!("Error");
});
}
}
#[derive(
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative,
)]
pub struct U8x19([u8; 19]);
direct_repr!(U8x19);
impl From<&[u8]> for U8x19 {
fn from(slice: &[u8]) -> Self {
let mut arr = Self::default();
arr.copy_from_slice(slice);
arr
}
}
#[derive(
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative,
)]
pub struct U8x31([u8; 31]);
direct_repr!(U8x31);
impl From<&[u8]> for U8x31 {
fn from(slice: &[u8]) -> Self {
let mut arr = Self::default();
arr.copy_from_slice(slice);
arr
}
}
pub fn databases_folder_path(folder: &str) -> String {
format!("{OUTPUTS_FOLDER_PATH}/databases/{folder}")
}
parser/src/databases/_trait.rs
+22 -10
View File
@@ -1,16 +1,23 @@
use std::{fs, io};
use std::{fs, io, path::PathBuf};
use crate::{
io::OUTPUTS_FOLDER_PATH,
structs::{Date, Height},
utils::log,
};
use super::{databases_folder_path, AnyDatabase};
use super::AnyDatabase;
pub trait AnyDatabaseGroup
where
Self: Sized,
{
fn init() -> Self {
let s = Self::import();
s.create_dir_all().unwrap();
s
}
fn import() -> Self;
fn folder<'a>() -> &'a str;
@@ -19,22 +26,27 @@ where
fn open_all(&mut self);
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()>;
// fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()>;
// fn defragment(&mut self);
fn create_dir_all(&self) -> color_eyre::Result<(), io::Error>;
fn remove_dir_all(&self) -> color_eyre::Result<(), io::Error> {
fs::remove_dir_all(Self::root())
}
fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
log(&format!("Reset {}", Self::folder()));
self.reset_metadata();
fs::remove_dir_all(Self::full_path())?;
self.remove_dir_all()?;
self.create_dir_all()?;
Ok(())
}
fn full_path() -> String {
databases_folder_path(Self::folder())
}
fn reset_metadata(&mut self);
fn root() -> PathBuf {
let folder = Self::folder();
PathBuf::from(format!("{OUTPUTS_FOLDER_PATH}/databases/{folder}"))
}
}
parser/src/databases/address_index_to_address_data.rs
+9 -3
View File
@@ -80,7 +80,9 @@ impl AddressIndexToAddressData {
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
);
Database::open(Self::folder(), &db_name).unwrap()
let path = Self::root().join(db_name);
Database::open(path).unwrap()
})
}
@@ -113,12 +115,16 @@ impl AddressIndexToAddressData {
impl AnyDatabaseGroup for AddressIndexToAddressData {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 1),
metadata: Metadata::import(Self::root(), 1),
map: BTreeMap::default(),
}
}
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
fs::create_dir_all(Self::root())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
@@ -128,7 +134,7 @@ impl AnyDatabaseGroup for AddressIndexToAddressData {
}
fn open_all(&mut self) {
let path = Self::full_path();
let path = Self::root();
let folder = fs::read_dir(path);
parser/src/databases/address_index_to_empty_address_data.rs
+9 -21
View File
@@ -80,7 +80,9 @@ impl AddressIndexToEmptyAddressData {
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
);
Database::open(Self::folder(), &db_name).unwrap()
let path = Self::root().join(db_name);
Database::open(path).unwrap()
})
}
@@ -92,12 +94,16 @@ impl AddressIndexToEmptyAddressData {
impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 1),
metadata: Metadata::import(Self::root(), 1),
map: BTreeMap::default(),
}
}
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
fs::create_dir_all(Self::root())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
@@ -107,7 +113,7 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
}
fn open_all(&mut self) {
let path = Self::full_path();
let path = Self::root();
let folder = fs::read_dir(path);
@@ -143,22 +149,4 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
self.metadata.export(height, date)
}
// fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
// self.drain_to_vec()
// .into_par_iter()
// .try_for_each(AnyDatabase::boxed_export)?;
// self.metadata.export(height, date)?;
// Ok(())
// }
// fn defragment(&mut self) {
// self.open_all();
// self.drain_to_vec()
// .into_par_iter()
// .for_each(AnyDatabase::boxed_defragment);
// }
}
parser/src/databases/address_to_address_index.rs
+83 -70
View File
@@ -1,13 +1,15 @@
use std::{collections::BTreeMap, fs, mem};
use std::{
collections::BTreeMap,
fs, mem,
path::{Path, PathBuf},
};
use allocative::Allocative;
use itertools::Itertools;
use crate::structs::{Address, Date, Height};
use crate::structs::{Address, Date, Height, U8x19, U8x31};
use super::{
databases_folder_path, AnyDatabase, AnyDatabaseGroup, Database, Metadata, U8x19, U8x31,
};
use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata};
type Value = u32;
type U8x19Database = Database<U8x19, Value>;
@@ -44,20 +46,6 @@ pub struct AddressToAddressIndex {
}
impl AddressToAddressIndex {
// pub fn safe_get(&mut self, address: &Address) -> Option<&Value> {
// match address {
// Address::Empty(key) => self.open_empty().get(key),
// Address::Unknown(key) => self.open_unknown().get(key),
// Address::MultiSig(key) => self.open_multisig().get(key),
// Address::P2PK((prefix, rest)) => self.open_p2pk(*prefix).get(rest),
// Address::P2PKH((prefix, rest)) => self.open_p2pkh(*prefix).get(rest),
// Address::P2SH((prefix, rest)) => self.open_p2sh(*prefix).get(rest),
// Address::P2WPKH((prefix, rest)) => self.open_p2wpkh(*prefix).get(rest),
// Address::P2WSH((prefix, rest)) => self.open_p2wsh(*prefix).get(rest),
// Address::P2TR((prefix, rest)) => self.open_p2tr(*prefix).get(rest),
// }
// }
pub fn open_db(&mut self, address: &Address) {
match address {
Address::Empty(_) => {
@@ -148,9 +136,7 @@ impl AddressToAddressIndex {
}
}
fn path_to_group_prefixes(path: &str) -> Vec<u16> {
let path = databases_folder_path(path);
fn path_to_group_prefixes(path: &Path) -> Vec<u16> {
let folder = fs::read_dir(path);
if folder.is_err() {
@@ -174,15 +160,16 @@ impl AddressToAddressIndex {
.collect_vec()
}
fn path_p2pk() -> String {
format!("{}/{}", Self::folder(), "p2pk")
fn path_p2pk() -> PathBuf {
Self::root().join("p2pk")
}
pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase {
let path = Self::path_p2pk();
self.p2pk
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
self.p2pk.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
})
}
fn open_all_p2pk(&mut self) {
@@ -190,21 +177,24 @@ impl AddressToAddressIndex {
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2pk
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
self.p2pk.insert(prefix, {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
});
});
}
fn path_p2pkh() -> String {
format!("{}/{}", Self::folder(), "p2pkh")
fn path_p2pkh() -> PathBuf {
Self::root().join("p2pkh")
}
pub fn open_p2pkh(&mut self, prefix: u16) -> &mut P2PKHDatabase {
let path = Self::path_p2pkh();
self.p2pkh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
self.p2pkh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
})
}
fn open_all_p2pkh(&mut self) {
@@ -212,21 +202,24 @@ impl AddressToAddressIndex {
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2pkh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
self.p2pkh.insert(prefix, {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
});
});
}
fn path_p2sh() -> String {
format!("{}/{}", Self::folder(), "p2sh")
fn path_p2sh() -> PathBuf {
Self::root().join("p2sh")
}
pub fn open_p2sh(&mut self, prefix: u16) -> &mut P2SHDatabase {
let path = Self::path_p2sh();
self.p2sh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
self.p2sh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
})
}
fn open_all_p2sh(&mut self) {
@@ -234,21 +227,24 @@ impl AddressToAddressIndex {
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2sh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
self.p2sh.insert(prefix, {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
});
});
}
fn path_p2wpkh() -> String {
format!("{}/{}", Self::folder(), "p2wpkh")
fn path_p2wpkh() -> PathBuf {
Self::root().join("p2wpkh")
}
pub fn open_p2wpkh(&mut self, prefix: u16) -> &mut P2WPKHDatabase {
let path = Self::path_p2wpkh();
self.p2wpkh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
self.p2wpkh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
})
}
fn open_all_p2wpkh(&mut self) {
@@ -256,21 +252,24 @@ impl AddressToAddressIndex {
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2wpkh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
self.p2wpkh.insert(prefix, {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
});
});
}
fn path_p2wsh() -> String {
format!("{}/{}", Self::folder(), "p2wsh")
fn path_p2wsh() -> PathBuf {
Self::root().join("p2wsh")
}
pub fn open_p2wsh(&mut self, prefix: u16) -> &mut P2WSHDatabase {
let path = Self::path_p2wsh();
self.p2wsh
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
self.p2wsh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
})
}
fn open_all_p2wsh(&mut self) {
@@ -278,21 +277,24 @@ impl AddressToAddressIndex {
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2wsh
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
self.p2wsh.insert(prefix, {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
});
});
}
fn path_p2tr() -> String {
format!("{}/{}", Self::folder(), "p2tr")
fn path_p2tr() -> PathBuf {
Self::root().join("p2tr")
}
pub fn open_p2tr(&mut self, prefix: u16) -> &mut P2TRDatabase {
let path = Self::path_p2tr();
self.p2tr
.entry(prefix)
.or_insert_with(|| Database::open(&path, &prefix.to_string()).unwrap())
self.p2tr.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
})
}
fn open_all_p2tr(&mut self) {
@@ -300,41 +302,43 @@ impl AddressToAddressIndex {
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
self.p2tr
.insert(prefix, Database::open(&path, &prefix.to_string()).unwrap());
self.p2tr.insert(prefix, {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
});
});
}
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
self.unknown
.get_or_insert_with(|| Database::open(Self::folder(), "unknown").unwrap())
.get_or_insert_with(|| Database::open(Self::root().join("unknown")).unwrap())
}
pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
self.op_return
.get_or_insert_with(|| Database::open(Self::folder(), "op_return").unwrap())
.get_or_insert_with(|| Database::open(Self::root().join("op_return")).unwrap())
}
pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
self.push_only
.get_or_insert_with(|| Database::open(Self::folder(), "push_only").unwrap())
.get_or_insert_with(|| Database::open(Self::root().join("push_only")).unwrap())
}
pub fn open_empty(&mut self) -> &mut UnknownDatabase {
self.empty
.get_or_insert_with(|| Database::open(Self::folder(), "empty").unwrap())
.get_or_insert_with(|| Database::open(Self::root().join("empty")).unwrap())
}
pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
self.multisig
.get_or_insert_with(|| Database::open(Self::folder(), "multisig").unwrap())
.get_or_insert_with(|| Database::open(Self::root().join("multisig")).unwrap())
}
}
impl AnyDatabaseGroup for AddressToAddressIndex {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 1),
metadata: Metadata::import(Self::root(), 1),
p2pk: BTreeMap::default(),
p2pkh: BTreeMap::default(),
@@ -350,6 +354,15 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
}
}
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
fs::create_dir_all(Self::path_p2pk()).unwrap();
fs::create_dir_all(Self::path_p2pkh()).unwrap();
fs::create_dir_all(Self::path_p2sh()).unwrap();
fs::create_dir_all(Self::path_p2wpkh()).unwrap();
fs::create_dir_all(Self::path_p2wsh()).unwrap();
fs::create_dir_all(Self::path_p2tr())
}
fn reset_metadata(&mut self) {
self.metadata.reset()
}
parser/src/databases/metadata.rs
+12 -17
View File
@@ -6,7 +6,7 @@ use std::{
fmt::Debug,
fs, io,
ops::{Deref, DerefMut},
path::Path,
path::{Path, PathBuf},
};
use crate::{
@@ -16,7 +16,7 @@ use crate::{
#[derive(Default, Debug, Encode, Decode, Allocative)]
pub struct Metadata {
path: String,
path: PathBuf,
data: MetadataData,
}
@@ -35,10 +35,10 @@ impl DerefMut for Metadata {
}
impl Metadata {
pub fn import(path: &str, version: u16) -> Self {
pub fn import(path: PathBuf, version: u16) -> Self {
Self {
path: path.to_owned(),
data: MetadataData::import(path, version),
data: MetadataData::import(&path, version),
path,
}
}
@@ -86,25 +86,20 @@ pub struct MetadataData {
}
impl MetadataData {
fn name<'a>() -> &'a str {
"metadata"
fn full_path(folder_path: &Path) -> PathBuf {
folder_path.join("metadata")
}
fn full_path(folder_path: &str) -> String {
let name = Self::name();
format!("{folder_path}/{name}")
}
pub fn import(path: &str, version: u16) -> Self {
pub fn import(path: &Path, version: u16) -> Self {
let mut s = Self::_import(path, version).unwrap_or_default();
s.version = version;
s
}
fn _import(path: &str, version: u16) -> color_eyre::Result<Self> {
fn _import(path: &Path, version: u16) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let s: MetadataData = Serialization::Binary.import(Path::new(&Self::full_path(path)))?;
let s: MetadataData = Serialization::Binary.import(path)?;
if s.version != version {
return Err(eyre!("Bad version"));
@@ -113,11 +108,11 @@ impl MetadataData {
Ok(s)
}
pub fn export(&self, path: &str) -> color_eyre::Result<()> {
pub fn export(&self, path: &Path) -> color_eyre::Result<()> {
Serialization::Binary.export(Path::new(&Self::full_path(path)), self)
}
pub fn reset(&mut self, path: &str) -> color_eyre::Result<(), io::Error> {
pub fn reset(&mut self, path: &Path) -> color_eyre::Result<(), io::Error> {
self.clear();
fs::remove_file(Self::full_path(path))
parser/src/databases/mod.rs
+6 -6
View File
@@ -43,17 +43,17 @@ pub struct Databases {
impl Databases {
pub fn import() -> Self {
let address_index_to_address_data = AddressIndexToAddressData::import();
let address_index_to_address_data = AddressIndexToAddressData::init();
let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::import();
let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init();
let address_to_address_index = AddressToAddressIndex::import();
let address_to_address_index = AddressToAddressIndex::init();
let txid_to_tx_data = TxidToTxData::import();
let txid_to_tx_data = TxidToTxData::init();
let txout_index_to_address_index = TxoutIndexToAddressIndex::import();
let txout_index_to_address_index = TxoutIndexToAddressIndex::init();
let txout_index_to_amount = TxoutIndexToAmount::import();
let txout_index_to_amount = TxoutIndexToAmount::init();
Self {
address_index_to_address_data,
parser/src/databases/txid_to_tx_data.rs
+12 -6
View File
@@ -8,9 +8,9 @@ use allocative::Allocative;
use biter::bitcoin::Txid;
use itertools::Itertools;
use crate::structs::{Date, Height, TxData};
use crate::structs::{Date, Height, TxData, U8x31};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, U8x31};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
type Key = U8x31;
type Value = TxData;
@@ -109,8 +109,10 @@ impl TxidToTxData {
#[inline(always)]
pub fn _open_db(&mut self, db_index: u16) -> &mut Database {
self.entry(db_index)
.or_insert_with(|| Database::open(Self::folder(), &db_index.to_string()).unwrap())
self.entry(db_index).or_insert_with(|| {
let path = Self::root().join(db_index.to_string());
Database::open(path).unwrap()
})
}
fn txid_to_key(txid: &Txid) -> U8x31 {
@@ -125,12 +127,16 @@ impl TxidToTxData {
impl AnyDatabaseGroup for TxidToTxData {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 2),
metadata: Metadata::import(Self::root(), 2),
map: BTreeMap::default(),
}
}
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
fs::create_dir_all(Self::root())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
@@ -140,7 +146,7 @@ impl AnyDatabaseGroup for TxidToTxData {
}
fn open_all(&mut self) {
let path = Self::full_path();
let path = Self::root();
let folder = fs::read_dir(path);
parser/src/databases/txout_index_to_address_index.rs
+8 -12
View File
@@ -77,7 +77,9 @@ impl TxoutIndexToAddressIndex {
(db_index + 1) * DB_MAX_SIZE
);
Database::open(Self::folder(), &db_name).unwrap()
let path = Self::root().join(db_name);
Database::open(path).unwrap()
})
}
@@ -89,21 +91,15 @@ impl TxoutIndexToAddressIndex {
impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 1),
metadata: Metadata::import(Self::root(), 1),
map: BTreeMap::default(),
}
}
// fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
// mem::take(&mut self.map)
// .into_par_iter()
// .try_for_each(|(_, db)| db.export())?;
// self.metadata.export(height, date)?;
// Ok(())
// }
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
fs::create_dir_all(Self::root())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
@@ -114,7 +110,7 @@ impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
}
fn open_all(&mut self) {
let path = Self::full_path();
let path = Self::root();
let folder = fs::read_dir(path);
parser/src/databases/txout_index_to_amount.rs
+9 -3
View File
@@ -77,7 +77,9 @@ impl TxoutIndexToAmount {
(db_index + 1) * DB_MAX_SIZE
);
Database::open(Self::folder(), &db_name).unwrap()
let path = Self::root().join(db_name);
Database::open(path).unwrap()
})
}
@@ -89,12 +91,16 @@ impl TxoutIndexToAmount {
impl AnyDatabaseGroup for TxoutIndexToAmount {
fn import() -> Self {
Self {
metadata: Metadata::import(&Self::full_path(), 1),
metadata: Metadata::import(Self::root(), 1),
map: BTreeMap::default(),
}
}
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
fs::create_dir_all(Self::root())
}
fn reset_metadata(&mut self) {
self.metadata.reset();
}
@@ -104,7 +110,7 @@ impl AnyDatabaseGroup for TxoutIndexToAmount {
}
fn open_all(&mut self) {
let path = Self::full_path();
let path = Self::root();
let folder = fs::read_dir(path);
parser/src/structs/address.rs
+1 -3
View File
@@ -1,9 +1,7 @@
use bitcoin_hashes::{hash160, Hash};
use biter::bitcoin::TxOut;
use crate::databases::{U8x19, U8x31};
use super::{AddressType, Counter};
use super::{AddressType, Counter, U8x19, U8x31};
#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq)]
pub enum Address {
parser/src/structs/array.rs
+31
View File
@@ -0,0 +1,31 @@
use std::fmt::Debug;
use allocative::Allocative;
use derive_deref::{Deref, DerefMut};
use sanakirja::{direct_repr, Storable, UnsizedStorable};
#[derive(
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative,
)]
pub struct U8x19([u8; 19]);
direct_repr!(U8x19);
impl From<&[u8]> for U8x19 {
fn from(slice: &[u8]) -> Self {
let mut arr = Self::default();
arr.copy_from_slice(slice);
arr
}
}
#[derive(
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative,
)]
pub struct U8x31([u8; 31]);
direct_repr!(U8x31);
impl From<&[u8]> for U8x31 {
fn from(slice: &[u8]) -> Self {
let mut arr = Self::default();
arr.copy_from_slice(slice);
arr
}
}
parser/src/structs/mod.rs
+2
View File
@@ -7,6 +7,7 @@ mod address_split;
mod address_type;
mod amount;
mod any_map;
mod array;
mod bi_map;
mod block_data;
mod block_path;
@@ -44,6 +45,7 @@ pub use address_split::*;
pub use address_type::*;
pub use amount::*;
pub use any_map::*;
pub use array::*;
pub use bi_map::*;
pub use block_data::*;
pub use block_path::*;