Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-02 02:20:00 -07:00
Code Issues Packages Projects Releases Wiki Activity

snkrj: move database struct to its own crate

Browse Source
This commit is contained in:
nym21
2024-12-14 14:55:44 +01:00
parent c883ed19d6
commit 50c77b51db
28 changed files with 702 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/parser/actions/iter_blocks.rs
View File

@@ -213,7 +213,9 @@ pub fn iter_blocks(
let defragment = is_safe
&& next_date_opt.is_some_and(|date| {
date.year() >= 2020 && date.is_january() && date.is_first_of_month()
(date.year() >= 2020 && date.is_january()
|| date.year() >= 2022 && date.is_june())
&& date.is_first_of_month()
});
export(ExportedData {

28
src/parser/actions/parse.rs
View File

@@ -241,7 +241,7 @@ pub fn parse(
databases
.txout_index_to_amount
.unsafe_insert(txout_index, amount);
.insert_to_ram(txout_index, amount);
if compute_addresses {
let address = address.unwrap();
@@ -253,7 +253,7 @@ pub fn parse(
if let Some(address_index) = address_index_opt.or_else(|| {
databases
.address_to_address_index
.unsafe_get_from_puts(&address)
.get_from_ram(&address)
.cloned()
}) {
let address_data = address_index_to_address_data
@@ -301,7 +301,7 @@ pub fn parse(
databases
.txout_index_to_address_index
.unsafe_insert(txout_index, address_index);
.insert_to_ram(txout_index, address_index);
}
});
@@ -336,9 +336,7 @@ pub fn parse(
.or_else(|| {
is_tx_data_from_cached_puts = true;
databases
.txid_to_tx_data
.unsafe_get_mut_from_puts(&input_txid)
databases.txid_to_tx_data.get_mut_from_ram(&input_txid)
});
// Can be none because 0 sats inputs happen
@@ -534,7 +532,7 @@ pub fn parse(
if remove_tx_data_from_cached_puts {
// Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
databases.txid_to_tx_data.remove_from_puts(&input_txid)
databases.txid_to_tx_data.remove_from_ram(&input_txid)
}
ControlFlow::Continue(())
@@ -558,7 +556,7 @@ pub fn parse(
txid_to_tx_data.into_iter().for_each(|(txid, tx_data)| {
if let Some(tx_data) = tx_data {
if tx_data.is_empty() {
databases.txid_to_tx_data.remove_from_db(txid);
databases.txid_to_tx_data.remove_later_from_disk(txid);
} else {
databases.txid_to_tx_data.update(txid, tx_data);
}
@@ -738,14 +736,14 @@ pub fn parse(
address_index_to_address_data.unwrap().into_iter().for_each(
|(address_index, address_data)| {
if address_data.is_empty() {
databases.address_index_to_empty_address_data.unsafe_insert(
databases.address_index_to_empty_address_data.insert_to_ram(
address_index,
EmptyAddressData::from_non_empty(&address_data),
);
} else {
databases
.address_index_to_address_data
.unsafe_insert(address_index, address_data);
.insert_to_ram(address_index, address_data);
}
},
)
@@ -901,7 +899,7 @@ fn prepare_inputs<'a>(
let mut tx_datas = txid_to_tx_data
.par_iter()
.map(|(txid, _)| txid_to_tx_data_db.unsafe_get(txid))
.map(|(txid, _)| txid_to_tx_data_db.get(txid))
.collect::<Vec<_>>();
txid_to_tx_data.values_mut().rev().for_each(|tx_data_opt| {
@@ -992,20 +990,20 @@ fn compute_address_index_to_address_data(
.par_iter_mut()
.for_each(|(address_index, address_data)| {
if let Some(_address_data) =
address_index_to_address_data_db.unsafe_get_from_cache(address_index)
address_index_to_address_data_db.get_from_ram(address_index)
{
_address_data.clone_into(address_data);
} else if let Some(empty_address_data) =
address_index_to_empty_address_data_db.unsafe_get_from_cache(address_index)
address_index_to_empty_address_data_db.get_from_ram(address_index)
{
*address_data = AddressData::from_empty(empty_address_data);
} else if let Some(_address_data) =
address_index_to_address_data_db.unsafe_get_from_db(address_index)
address_index_to_address_data_db.get_from_disk(address_index)
{
_address_data.clone_into(address_data);
} else {
let empty_address_data = address_index_to_empty_address_data_db
.unsafe_get_from_db(address_index)
.get_from_disk(address_index)
.unwrap();
*address_data = AddressData::from_empty(empty_address_data);

229
src/parser/databases/_database.rs
View File

@@ -1,229 +0,0 @@
// https://docs.rs/sanakirja/latest/sanakirja/index.html
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
use std::{
collections::{BTreeMap, BTreeSet},
fmt::Debug,
fs, io, mem,
path::PathBuf,
};
use allocative::Allocative;
use sanakirja::{
btree::{self, page, Db_, Iter},
Commit, Env, Error, MutTxn, RootDb, Storable,
};
///
/// Simple wrapper around Sanakirja Database with cached puts and dels for safe use outside exports.
///
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching "gets" along the way.
///
#[derive(Allocative)]
#[allocative(bound = "Key: Allocative, Value: Allocative")]
pub struct Database<Key, Value>
where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq,
{
path: PathBuf,
cached_puts: BTreeMap<Key, Value>,
cached_dels: BTreeSet<Key>,
#[allocative(skip)]
db: Db_<Key, Value, page::Page<Key, Value>>,
#[allocative(skip)]
txn: MutTxn<Env, ()>,
}
const ROOT_DB: usize = 0;
const PAGE_SIZE: u64 = 4096;
impl<Key, Value> Database<Key, Value>
where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq,
{
pub fn open(path: PathBuf) -> color_eyre::Result<Self> {
let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? };
let mut txn = Env::mut_txn_begin(env)?;
let db = txn
.root_db(ROOT_DB)
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
Ok(Self {
path,
cached_puts: BTreeMap::default(),
cached_dels: BTreeSet::default(),
db,
txn,
})
}
#[inline]
pub fn iter(&self) -> Iter<'_, MutTxn<Env, ()>, Key, Value, page::Page<Key, Value>> {
btree::iter(&self.txn, &self.db, None).unwrap()
}
pub fn collect(&self) -> BTreeMap<Key, Value>
where
Value: Clone,
{
self.iter()
.map(|r| r.unwrap())
.map(|(key, value)| (key.clone(), value.clone()))
.collect::<_>()
}
#[inline]
pub fn get(&self, key: &Key) -> Option<&Value> {
if let Some(cached_put) = self.get_from_puts(key) {
return Some(cached_put);
}
self.db_get(key)
}
#[inline]
pub fn db_get(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
if let Some((key_found, v)) = option {
if key == key_found {
return Some(v);
}
}
None
}
#[inline]
pub fn get_from_puts(&self, key: &Key) -> Option<&Value> {
self.cached_puts.get(key)
}
#[inline]
pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> {
self.cached_puts.get_mut(key)
}
#[inline]
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.remove_from_puts(key).or_else(|| {
self.db_remove(key);
None
})
}
#[inline]
pub fn db_remove(&mut self, key: &Key) {
self.cached_dels.insert(key.clone());
}
#[inline]
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.insert(key.clone());
self.cached_puts.insert(key, value)
}
#[inline]
pub fn is_empty(&self) -> bool {
self.iter().next().is_none()
}
#[inline]
pub fn remove_from_puts(&mut self, key: &Key) -> Option<Value> {
self.cached_puts.remove(key)
}
#[inline]
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_dels.remove(&key);
self.unsafe_insert(key, value)
}
#[inline]
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.cached_puts.insert(key, value)
}
fn db_multi_put(&mut self, tree: BTreeMap<Key, Value>) -> Result<(), Error> {
tree.into_iter()
.try_for_each(|(key, value)| -> Result<(), Error> {
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
})
}
fn db_multi_del(&mut self, tree: BTreeSet<Key>) -> Result<(), Error> {
tree.into_iter().try_for_each(|key| -> Result<(), Error> {
btree::del(&mut self.txn, &mut self.db, &key, None)?;
Ok(())
})
}
}
pub trait AnyDatabase {
#[allow(unused)]
fn export(self, defragment: bool) -> color_eyre::Result<(), Error>;
fn boxed_export(self: Box<Self>, defragment: bool) -> color_eyre::Result<(), Error>;
#[allow(unused)]
fn destroy(self) -> io::Result<()>;
}
impl<Key, Value> AnyDatabase for Database<Key, Value>
where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq + Clone,
{
fn export(self, defragment: bool) -> color_eyre::Result<(), Error> {
Box::new(self).boxed_export(defragment)
}
fn boxed_export(mut self: Box<Self>, defragment: bool) -> color_eyre::Result<(), Error> {
if defragment {
let mut btree = self.as_ref().collect();
let path = self.path.to_owned();
self.cached_dels.iter().for_each(|key| {
btree.remove(key);
});
btree.append(&mut self.cached_puts);
self.destroy()?;
*self = Self::open(path).unwrap();
if !self.is_empty() {
panic!()
}
self.cached_puts = btree;
}
if self.cached_dels.is_empty() && self.cached_puts.is_empty() {
return Ok(());
}
let cached_dels = mem::take(&mut self.cached_dels);
self.db_multi_del(cached_dels)?;
let cached_puts = mem::take(&mut self.cached_puts);
self.db_multi_put(cached_puts)?;
self.txn.set_root(ROOT_DB, self.db.db.into());
self.txn.commit()
}
fn destroy(self) -> io::Result<()> {
let path = self.path.to_owned();
drop(self);
fs::remove_file(&path)
}
}

3
src/parser/databases/_trait.rs
View File

@@ -1,10 +1,11 @@
use std::{fs, io, path::Path};
use log::info;
use snkrj::AnyDatabase;
use crate::structs::{Config, Date, Height};
use super::{AnyDatabase, Metadata};
use super::Metadata;
pub trait AnyDatabaseGroup
where

18
src/parser/databases/address_index_to_address_data.rs
View File

@@ -8,6 +8,7 @@ use std::{
use allocative::Allocative;
use itertools::Itertools;
use rayon::prelude::*;
use snkrj::{AnyDatabase, Database as _Database};
use crate::{
parser::states::AddressCohortsDurableStates,
@@ -15,7 +16,7 @@ use crate::{
utils::time,
};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
use super::{AnyDatabaseGroup, Metadata};
type Key = u32;
type Value = AddressData;
@@ -25,6 +26,7 @@ type Database = _Database<Key, Value>;
pub struct AddressIndexToAddressData {
path: PathBuf,
pub metadata: Metadata,
#[allocative(skip)]
pub map: BTreeMap<usize, Database>,
}
@@ -45,10 +47,10 @@ impl DerefMut for AddressIndexToAddressData {
pub const ADDRESS_INDEX_DB_MAX_SIZE: usize = 250_000;
impl AddressIndexToAddressData {
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
self.metadata.called_insert();
self.open_db(&key).unsafe_insert(key, value)
self.open_db(&key).insert_to_ram(key, value)
}
pub fn remove(&mut self, key: &Key) -> Option<Value> {
@@ -59,16 +61,16 @@ impl AddressIndexToAddressData {
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
/// Though it makes it easy to use with rayon.
pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
self.get(&db_index).unwrap().get_from_puts(key)
self.get(&db_index).unwrap().get_from_ram(key)
}
pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
self.get(&db_index).unwrap().db_get(key)
self.get(&db_index).unwrap().get_from_disk(key)
}
pub fn open_db(&mut self, key: &Key) -> &mut Database {
@@ -99,7 +101,7 @@ impl AddressIndexToAddressData {
let mut s = AddressCohortsDurableStates::default();
database
.iter()
.iter_disk()
.map(|r| r.unwrap().1)
.for_each(|address_data| s.increment(address_data).unwrap());

18
src/parser/databases/address_index_to_empty_address_data.rs
View File

@@ -7,12 +7,11 @@ use std::{
use allocative::Allocative;
use itertools::Itertools;
use snkrj::{AnyDatabase, Database as _Database};
use crate::structs::{Config, EmptyAddressData};
use super::{
AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE,
};
use super::{AnyDatabaseGroup, Metadata, ADDRESS_INDEX_DB_MAX_SIZE};
type Key = u32;
type Value = EmptyAddressData;
@@ -22,6 +21,7 @@ type Database = _Database<Key, Value>;
pub struct AddressIndexToEmptyAddressData {
path: PathBuf,
pub metadata: Metadata,
#[allocative(skip)]
map: BTreeMap<usize, Database>,
}
@@ -40,10 +40,10 @@ impl DerefMut for AddressIndexToEmptyAddressData {
}
impl AddressIndexToEmptyAddressData {
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
self.metadata.called_insert();
self.open_db(&key).unsafe_insert(key, value)
self.open_db(&key).insert_to_ram(key, value)
}
pub fn remove(&mut self, key: &Key) -> Option<Value> {
@@ -54,13 +54,13 @@ impl AddressIndexToEmptyAddressData {
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
/// Though it makes it easy to use with rayon.
pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
self.get(&db_index).and_then(|db| db.get_from_puts(key))
self.get(&db_index).and_then(|db| db.get_from_ram(key))
}
pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
self.get(&db_index)
@@ -68,7 +68,7 @@ impl AddressIndexToEmptyAddressData {
dbg!(&self.map.keys(), &key, &db_index);
panic!()
})
.db_get(key)
.get_from_disk(key)
}
pub fn open_db(&mut self, key: &Key) -> &mut Database {

38
src/parser/databases/address_to_address_index.rs
View File

@@ -6,10 +6,11 @@ use std::{
use allocative::Allocative;
use itertools::Itertools;
use snkrj::{AnyDatabase, Database};
use crate::structs::{Address, Config, U8x19, U8x31};
use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata};
use super::{AnyDatabaseGroup, Metadata};
type Value = u32;
type U8x19Database = Database<U8x19, Value>;
@@ -33,16 +34,27 @@ pub struct AddressToAddressIndex {
path: PathBuf,
pub metadata: Metadata,
#[allocative(skip)]
p2pk: BTreeMap<u16, P2PKDatabase>,
#[allocative(skip)]
p2pkh: BTreeMap<u16, P2PKHDatabase>,
#[allocative(skip)]
p2sh: BTreeMap<u16, P2SHDatabase>,
#[allocative(skip)]
p2wpkh: BTreeMap<u16, P2WPKHDatabase>,
#[allocative(skip)]
p2wsh: BTreeMap<u16, P2WSHDatabase>,
#[allocative(skip)]
p2tr: BTreeMap<u16, P2TRDatabase>,
#[allocative(skip)]
op_return: Option<OpReturnDatabase>,
#[allocative(skip)]
push_only: Option<PushOnlyDatabase>,
#[allocative(skip)]
unknown: Option<UnknownDatabase>,
#[allocative(skip)]
empty: Option<EmptyDatabase>,
#[allocative(skip)]
multisig: Option<MultisigDatabase>,
}
@@ -103,19 +115,19 @@ impl AddressToAddressIndex {
}
}
pub fn unsafe_get_from_puts(&self, address: &Address) -> Option<&Value> {
pub fn get_from_ram(&self, address: &Address) -> Option<&Value> {
match address {
Address::Empty(key) => self.empty.as_ref().unwrap().get_from_puts(key),
Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_puts(key),
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_puts(key),
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_puts(key),
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_puts(key),
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_puts(key),
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_puts(key),
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_puts(key),
Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_puts(key),
Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_puts(key),
Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_puts(key),
Address::Empty(key) => self.empty.as_ref().unwrap().get_from_ram(key),
Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_ram(key),
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_ram(key),
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_ram(key),
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_ram(key),
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_ram(key),
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_ram(key),
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_ram(key),
Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_ram(key),
Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_ram(key),
Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_ram(key),
}
}

3
src/parser/databases/mod.rs
View File

@@ -1,6 +1,5 @@
use allocative::Allocative;
mod _database;
mod _trait;
mod address_index_to_address_data;
mod address_index_to_empty_address_data;
@@ -10,7 +9,6 @@ mod txid_to_tx_data;
mod txout_index_to_address_index;
mod txout_index_to_amount;
pub use _database::*;
use _trait::*;
pub use address_index_to_address_data::*;
pub use address_index_to_empty_address_data::*;
@@ -19,6 +17,7 @@ use itertools::Itertools;
use log::info;
use metadata::*;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use snkrj::AnyDatabase;
pub use txid_to_tx_data::*;
pub use txout_index_to_address_index::*;
pub use txout_index_to_amount::*;

53
src/parser/databases/txid_to_tx_data.rs
View File

@@ -1,17 +1,17 @@
use std::{
collections::BTreeMap,
fs, mem,
ops::{Deref, DerefMut},
path::{Path, PathBuf},
};
use allocative::Allocative;
use biter::bitcoin::Txid;
use itertools::Itertools;
use snkrj::{AnyDatabase, Database as _Database};
use crate::structs::{Config, TxData, U8x31};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
use super::{AnyDatabaseGroup, Metadata};
type Key = U8x31;
type Value = TxData;
@@ -21,23 +21,10 @@ type Database = _Database<Key, Value>;
pub struct TxidToTxData {
path: PathBuf,
pub metadata: Metadata,
#[allocative(skip)]
map: BTreeMap<u16, Database>,
}
impl Deref for TxidToTxData {
type Target = BTreeMap<u16, Database>;
fn deref(&self) -> &Self::Target {
&self.map
}
}
impl DerefMut for TxidToTxData {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.map
}
}
impl TxidToTxData {
pub fn insert(&mut self, txid: &Txid, tx_index: Value) -> Option<Value> {
self.metadata.called_insert();
@@ -47,53 +34,41 @@ impl TxidToTxData {
self.open_db(txid).insert(txid_key, tx_index)
}
// pub fn safe_get(&mut self, txid: &Txid) -> Option<&Value> {
// let txid_key = Self::txid_to_key(txid);
// self.open_db(txid).get(&txid_key)
// }
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed.
/// Though it makes it easy to use with rayon
pub fn unsafe_get(&self, txid: &Txid) -> Option<&Value> {
pub fn get(&self, txid: &Txid) -> Option<&Value> {
let txid_key = Self::txid_to_key(txid);
let db_index = Self::db_index(txid);
self.get(&db_index).unwrap().get(&txid_key)
self.map.get(&db_index).unwrap().get(&txid_key)
}
// pub fn unsafe_get_from_puts(&self, txid: &Txid) -> Option<&Value> {
// let txid_key = Self::txid_to_key(txid);
// let db_index = Self::db_index(txid);
// self.get(&db_index).unwrap().get_from_puts(&txid_key)
// }
pub fn unsafe_get_mut_from_puts(&mut self, txid: &Txid) -> Option<&mut Value> {
pub fn get_mut_from_ram(&mut self, txid: &Txid) -> Option<&mut Value> {
let txid_key = Self::txid_to_key(txid);
let db_index = Self::db_index(txid);
self.get_mut(&db_index)
self.map
.get_mut(&db_index)
.unwrap()
.get_mut_from_puts(&txid_key)
.get_mut_from_ram(&txid_key)
}
pub fn remove_from_db(&mut self, txid: &Txid) {
pub fn remove_later_from_disk(&mut self, txid: &Txid) {
self.metadata.called_remove();
let txid_key = Self::txid_to_key(txid);
self.open_db(txid).db_remove(&txid_key);
self.open_db(txid).remove_later_from_disk(&txid_key);
}
pub fn remove_from_puts(&mut self, txid: &Txid) {
pub fn remove_from_ram(&mut self, txid: &Txid) {
self.metadata.called_remove();
let txid_key = Self::txid_to_key(txid);
self.open_db(txid).remove_from_puts(&txid_key);
self.open_db(txid).remove_from_ram(&txid_key);
}
pub fn update(&mut self, txid: &Txid, tx_data: TxData) {
@@ -111,7 +86,7 @@ impl TxidToTxData {
#[inline(always)]
pub fn _open_db(&mut self, db_index: u16) -> &mut Database {
let path = self.path.to_owned();
self.entry(db_index).or_insert_with(|| {
self.map.entry(db_index).or_insert_with(|| {
let path = path.join(db_index.to_string());
Database::open(path).unwrap()
})

16
src/parser/databases/txout_index_to_address_index.rs
View File

@@ -7,10 +7,11 @@ use std::{
use allocative::Allocative;
use itertools::Itertools;
use snkrj::{AnyDatabase, Database as _Database};
use crate::structs::{Config, TxoutIndex};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
use super::{AnyDatabaseGroup, Metadata};
type Key = TxoutIndex;
type Value = u32;
@@ -20,6 +21,7 @@ type Database = _Database<Key, Value>;
pub struct TxoutIndexToAddressIndex {
path: PathBuf,
pub metadata: Metadata,
#[allocative(skip)]
map: BTreeMap<usize, Database>,
}
@@ -40,20 +42,12 @@ impl DerefMut for TxoutIndexToAddressIndex {
const DB_MAX_SIZE: usize = 10_000_000_000;
impl TxoutIndexToAddressIndex {
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
self.metadata.called_insert();
self.open_db(&key).unsafe_insert(key, value)
self.open_db(&key).insert_to_ram(key, value)
}
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
// self.open_db(key).remove_from_puts(key).map(|v| {
// self.metadata.called_remove();
// v
// })
// }
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.metadata.called_remove();

16
src/parser/databases/txout_index_to_amount.rs
View File

@@ -7,10 +7,11 @@ use std::{
use allocative::Allocative;
use itertools::Itertools;
use snkrj::{AnyDatabase, Database as _Database};
use crate::structs::{Amount, Config, TxoutIndex};
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
use super::{AnyDatabaseGroup, Metadata};
type Key = TxoutIndex;
type Value = Amount;
@@ -20,6 +21,7 @@ type Database = _Database<Key, Value>;
pub struct TxoutIndexToAmount {
path: PathBuf,
pub metadata: Metadata,
#[allocative(skip)]
map: BTreeMap<usize, Database>,
}
@@ -40,20 +42,12 @@ impl DerefMut for TxoutIndexToAmount {
const DB_MAX_SIZE: usize = 10_000_000_000;
impl TxoutIndexToAmount {
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
self.metadata.called_insert();
self.open_db(&key).unsafe_insert(key, value)
self.open_db(&key).insert_to_ram(key, value)
}
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
// self.open_db(key).remove_from_puts(key).map(|v| {
// self.metadata.called_remove();
// v
// })
// }
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.metadata.called_remove();

2
src/structs/address_data.rs
View File

@@ -1,6 +1,6 @@
use allocative::Allocative;
use color_eyre::eyre::eyre;
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::{AddressType, Amount, EmptyAddressData, LiquidityClassification, Price};

2
src/structs/amount.rs
View File

@@ -12,8 +12,8 @@ use bincode::{
};
use biter::bitcoin::Amount as BitcoinAmount;
use derive_deref::{Deref, DerefMut};
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use serde::{Deserialize, Serialize};
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::Height;

2
src/structs/array.rs
View File

@@ -2,7 +2,7 @@ use std::fmt::Debug;
use allocative::Allocative;
use derive_deref::{Deref, DerefMut};
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative,

4
src/structs/date.rs
View File

@@ -74,6 +74,10 @@ impl Date {
self.month() == 1
}
pub fn is_june(&self) -> bool {
self.month() == 6
}
pub fn is_first_of_month(&self) -> bool {
self.day() == 1
}

2
src/structs/empty_address_data.rs
View File

@@ -1,5 +1,5 @@
use allocative::Allocative;
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::{AddressData, AddressType, Amount};

2
src/structs/tx_data.rs
View File

@@ -1,5 +1,5 @@
use allocative::Allocative;
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::BlockPath;

2
src/structs/txout_index.rs
View File

@@ -1,6 +1,6 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Encode, Decode, Allocative)]
pub struct TxoutIndex {