From 50c77b51db8214680f548ee9efcf7ba8ebcf6388 Mon Sep 17 00:00:00 2001 From: nym21 Date: Sat, 14 Dec 2024 14:55:44 +0100 Subject: [PATCH] snkrj: move database struct to its own crate --- CHANGELOG.md | 3 +- Cargo.lock | 9 +- Cargo.toml | 2 +- crates/biter/LICENSE.md | 2 +- crates/snkrj/Cargo.lock | 242 +++++++++++++++++ crates/snkrj/Cargo.toml | 12 + crates/snkrj/LICENSE.md | 21 ++ crates/snkrj/README.md | 37 +++ crates/snkrj/src/lib.rs | 252 ++++++++++++++++++ crates/snkrj/src/main.rs | 29 ++ src/parser/actions/iter_blocks.rs | 4 +- src/parser/actions/parse.rs | 28 +- src/parser/databases/_database.rs | 229 ---------------- src/parser/databases/_trait.rs | 3 +- .../address_index_to_address_data.rs | 18 +- .../address_index_to_empty_address_data.rs | 18 +- .../databases/address_to_address_index.rs | 38 ++- src/parser/databases/mod.rs | 3 +- src/parser/databases/txid_to_tx_data.rs | 53 +--- .../databases/txout_index_to_address_index.rs | 16 +- src/parser/databases/txout_index_to_amount.rs | 16 +- src/structs/address_data.rs | 2 +- src/structs/amount.rs | 2 +- src/structs/array.rs | 2 +- src/structs/date.rs | 4 + src/structs/empty_address_data.rs | 2 +- src/structs/tx_data.rs | 2 +- src/structs/txout_index.rs | 2 +- 28 files changed, 702 insertions(+), 349 deletions(-) create mode 100644 crates/snkrj/Cargo.lock create mode 100644 crates/snkrj/Cargo.toml create mode 100644 crates/snkrj/LICENSE.md create mode 100644 crates/snkrj/README.md create mode 100644 crates/snkrj/src/lib.rs create mode 100644 crates/snkrj/src/main.rs delete mode 100644 src/parser/databases/_database.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b4a75e0..339cc3a3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,12 @@ # v0.6.0 | WIP - Merged parser and server crates into a single project (and thus executable) -- Started using `log` and `env_logger` crates +- Started using `log` and `env_logger` crates instead of custom code - Improved logs - Added `--server BOOL` and `--parser BOOL` parameters (both are true by default) - Automated databases defragmention (and removed parameter) - Fixed input being unfocused right after being focused in Brave browser +- Moved Sanakirja database wrapper to its own crate: `snkrj` # [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04 diff --git a/Cargo.lock b/Cargo.lock index cd0f6c81f..e72dc3740 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,9 +1524,9 @@ dependencies = [ "regex", "reqwest", "rlimit", - "sanakirja", "serde", "serde_json", + "snkrj", "struct_iterable", "swc", "swc_common", @@ -2621,6 +2621,13 @@ dependencies = [ "version_check", ] +[[package]] +name = "snkrj" +version = "0.1.0" +dependencies = [ + "sanakirja", +] + [[package]] name = "socket2" version = "0.5.7" diff --git a/Cargo.toml b/Cargo.toml index bd7c6bde3..19d50bc1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ rayon = "1.10.0" regex = "1.11.1" reqwest = { version = "0.12.9", features = ["blocking", "json"] } rlimit = "0.10.2" -sanakirja = "1.4.3" +snkrj = { path = "./crates/snkrj" } serde = { version = "1.0.216", features = ["derive"] } serde_json = "1.0.133" struct_iterable = { path = "./crates/iterable" } diff --git a/crates/biter/LICENSE.md b/crates/biter/LICENSE.md index 690c7b5da..3fd127e58 100644 --- a/crates/biter/LICENSE.md +++ b/crates/biter/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Biter +Copyright (c) 2024 biter Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/crates/snkrj/Cargo.lock b/crates/snkrj/Cargo.lock new file mode 100644 index 000000000..458967873 --- /dev/null +++ b/crates/snkrj/Cargo.lock @@ -0,0 +1,242 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "libc" +version = "0.2.168" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "sanakirja" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81aaf70d064e2122209f04d01fd91e8908e7a327b516236e1cbc0c3f34ac6d11" +dependencies = [ + "fs2", + "log", + "memmap2", + "parking_lot", + "sanakirja-core", + "serde", + "thiserror", +] + +[[package]] +name = "sanakirja-core" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "snkrj" +version = "0.1.0" +dependencies = [ + "sanakirja", +] + +[[package]] +name = "syn" +version = "2.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/crates/snkrj/Cargo.toml b/crates/snkrj/Cargo.toml new file mode 100644 index 000000000..864b234b6 --- /dev/null +++ b/crates/snkrj/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "snkrj" +description = "A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap" +version = "0.1.0" +license = "MIT" +repository = "https://github.com/kibo-money/kibo/tree/main/crates/snkrj" +keywords = ["database", "sanakirja", "btreemap"] +categories = ["database"] +edition = "2021" + +[dependencies] +sanakirja = "1.4.3" diff --git a/crates/snkrj/LICENSE.md b/crates/snkrj/LICENSE.md new file mode 100644 index 000000000..aac235dcb --- /dev/null +++ b/crates/snkrj/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 snkrj + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/snkrj/README.md b/crates/snkrj/README.md new file mode 100644 index 000000000..1803d180c --- /dev/null +++ b/crates/snkrj/README.md @@ -0,0 +1,37 @@ +# snkrj + +A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap. + +## Example + +```rust +use snkrj::{AnyDatabase, Database}; + +fn main() { + let path = std::env::temp_dir().join("./db"); + + let database: Database = Database::open(path.clone()).unwrap(); + let _ = database.destroy(); + + let mut database: Database = Database::open(path.clone()).unwrap(); + database.insert(64, 128); + database.export(false).unwrap(); + + let mut database: Database = Database::open(path).unwrap(); + database.insert(1, 2); + database.insert(128, 256); + println!("iter_ram:"); + database.iter_ram().for_each(|pair| { + println!("{:?}", pair); + }); + println!("iter_disk:"); + database.iter_disk().for_each(|pair| { + println!("{:?}", pair.unwrap()); + }); + println!("iter_ram_then_disk:"); + database.iter_ram_then_disk().for_each(|pair| { + println!("{:?}", pair); + }); + database.export(false).unwrap(); +} +``` diff --git a/crates/snkrj/src/lib.rs b/crates/snkrj/src/lib.rs new file mode 100644 index 000000000..a99598595 --- /dev/null +++ b/crates/snkrj/src/lib.rs @@ -0,0 +1,252 @@ +// https://docs.rs/sanakirja/latest/sanakirja/index.html +// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Debug, + fs, io, mem, + path::PathBuf, + result::Result, +}; + +use sanakirja::btree::{page, Db_}; +pub use sanakirja::*; + +/// +/// A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap. +/// +/// The state of the tree is uncommited until `.export()` is called during which it is unsafe to stop the program. +/// +pub struct Database +where + Key: Ord + Clone + Debug + Storable, + Value: Storable + PartialEq, +{ + path: PathBuf, + puts: BTreeMap, + dels: BTreeSet, + db: Db_>, + txn: MutTxn, +} + +const ROOT_DB: usize = 0; +const PAGE_SIZE: u64 = 4096; + +impl Database +where + Key: Ord + Clone + Debug + Storable, + Value: Storable + PartialEq, +{ + /// Open a database without a lock file where only one instance is safe to open. + pub fn open(path: PathBuf) -> Result { + let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? }; + + let mut txn = Env::mut_txn_begin(env)?; + + let db = txn + .root_db(ROOT_DB) + .unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() }); + + Ok(Self { + path, + puts: BTreeMap::default(), + dels: BTreeSet::default(), + db, + txn, + }) + } + + #[inline] + pub fn get(&self, key: &Key) -> Option<&Value> { + if let Some(cached_put) = self.get_from_ram(key) { + return Some(cached_put); + } + + self.get_from_disk(key) + } + + /// Get only from the uncommited tree (ram) without checking the database (disk) + #[inline] + pub fn get_from_ram(&self, key: &Key) -> Option<&Value> { + self.puts.get(key) + } + + /// Get mut only from the uncommited tree (ram) without checking the database (disk) + #[inline] + pub fn get_mut_from_ram(&mut self, key: &Key) -> Option<&mut Value> { + self.puts.get_mut(key) + } + + /// Get only from the database (disk) without checking the uncommited tree (ram) + #[inline] + pub fn get_from_disk(&self, key: &Key) -> Option<&Value> { + let option = btree::get(&self.txn, &self.db, key, None).unwrap(); + + if let Some((key_found, v)) = option { + if key == key_found { + return Some(v); + } + } + + None + } + + #[inline] + pub fn insert(&mut self, key: Key, value: Value) -> Option { + self.dels.remove(&key); + self.insert_to_ram(key, value) + } + + /// Insert without removing the key to the dels tree, so be sure that it hasn't added to the delete set + #[inline] + pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option { + self.puts.insert(key, value) + } + + #[inline] + pub fn update(&mut self, key: Key, value: Value) -> Option { + self.dels.insert(key.clone()); + self.puts.insert(key, value) + } + + #[inline] + pub fn remove(&mut self, key: &Key) -> Option { + self.remove_from_ram(key).or_else(|| { + self.remove_later_from_disk(key); + + None + }) + } + + /// Get only from the uncommited tree (ram) without checking the database (disk) + #[inline] + pub fn remove_from_ram(&mut self, key: &Key) -> Option { + self.puts.remove(key) + } + + /// Add the key only to the dels tree without checking if it's present in the puts tree, only use if you are positive that you neither added nor updated an entry with this key + #[inline] + pub fn remove_later_from_disk(&mut self, key: &Key) { + self.dels.insert(key.clone()); + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.iter_disk().next().is_none() + } + + /// Iterate over key/value pairs from the uncommited tree (ram) + #[inline] + pub fn iter_ram(&self) -> std::collections::btree_map::Iter<'_, Key, Value> { + self.puts.iter() + } + + /// Iterate over key/value pairs from the database (disk) + #[inline] + pub fn iter_disk( + &self, + ) -> btree::Iter<'_, MutTxn, Key, Value, page::Page> { + btree::iter(&self.txn, &self.db, None).unwrap() + } + + /// Iterate over key/value pairs + #[inline] + pub fn iter_ram_then_disk(&self) -> impl Iterator { + self.iter_ram().chain(self.iter_disk().map(|r| r.unwrap())) + } + + /// Collect a **clone** of all uncommited key/value pairs (ram) + pub fn collect_ram(&self) -> BTreeMap + where + Value: Clone, + { + self.puts.clone() + } + + /// Collect a **clone** of all key/value pairs from the database (disk) + pub fn collect_disk(&self) -> BTreeMap + where + Value: Clone, + { + self.iter_disk() + .map(|r| r.unwrap()) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::<_>() + } +} + +pub trait AnyDatabase { + #[allow(unused)] + fn export(self, defragment: bool) -> Result<(), Error>; + fn boxed_export(self: Box, defragment: bool) -> Result<(), Error>; + #[allow(unused)] + fn destroy(self) -> io::Result<()>; +} + +impl AnyDatabase for Database +where + Key: Ord + Clone + Debug + Storable, + Value: Storable + PartialEq + Clone, +{ + /// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space + /// + /// /!\ Do not kill the program while this function is runnning /!\ + fn export(self, defragment: bool) -> Result<(), Error> { + Box::new(self).boxed_export(defragment) + } + + /// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space + /// + /// /!\ Do not kill the program while this function is runnning /!\ + fn boxed_export(mut self: Box, defragment: bool) -> Result<(), Error> { + if defragment { + let mut btree = self.as_ref().collect_disk(); + + let path = self.path.to_owned(); + self.dels.iter().for_each(|key| { + btree.remove(key); + }); + btree.append(&mut self.puts); + + self.destroy()?; + + *self = Self::open(path).unwrap(); + + if !self.is_empty() { + panic!() + } + + self.puts = btree; + } + + if self.dels.is_empty() && self.puts.is_empty() { + return Ok(()); + } + + mem::take(&mut self.dels) + .into_iter() + .try_for_each(|key| -> Result<(), Error> { + btree::del(&mut self.txn, &mut self.db, &key, None)?; + Ok(()) + })?; + + mem::take(&mut self.puts).into_iter().try_for_each( + |(key, value)| -> Result<(), Error> { + btree::put(&mut self.txn, &mut self.db, &key, &value)?; + Ok(()) + }, + )?; + + self.txn.set_root(ROOT_DB, self.db.db.into()); + + self.txn.commit() + } + + fn destroy(self) -> io::Result<()> { + let path = self.path.to_owned(); + + drop(self); + + fs::remove_file(&path) + } +} diff --git a/crates/snkrj/src/main.rs b/crates/snkrj/src/main.rs new file mode 100644 index 000000000..da3c39d23 --- /dev/null +++ b/crates/snkrj/src/main.rs @@ -0,0 +1,29 @@ +use snkrj::{AnyDatabase, Database}; + +fn main() { + let path = std::env::temp_dir().join("./db"); + + let database: Database = Database::open(path.clone()).unwrap(); + let _ = database.destroy(); + + let mut database: Database = Database::open(path.clone()).unwrap(); + database.insert(64, 128); + database.export(false).unwrap(); + + let mut database: Database = Database::open(path).unwrap(); + database.insert(1, 2); + database.insert(128, 256); + println!("iter_ram:"); + database.iter_ram().for_each(|pair| { + println!("{:?}", pair); + }); + println!("iter_disk:"); + database.iter_disk().for_each(|pair| { + println!("{:?}", pair.unwrap()); + }); + println!("iter_ram_then_disk:"); + database.iter_ram_then_disk().for_each(|pair| { + println!("{:?}", pair); + }); + database.export(false).unwrap(); +} diff --git a/src/parser/actions/iter_blocks.rs b/src/parser/actions/iter_blocks.rs index 84e5a4af4..fcd41d254 100644 --- a/src/parser/actions/iter_blocks.rs +++ b/src/parser/actions/iter_blocks.rs @@ -213,7 +213,9 @@ pub fn iter_blocks( let defragment = is_safe && next_date_opt.is_some_and(|date| { - date.year() >= 2020 && date.is_january() && date.is_first_of_month() + (date.year() >= 2020 && date.is_january() + || date.year() >= 2022 && date.is_june()) + && date.is_first_of_month() }); export(ExportedData { diff --git a/src/parser/actions/parse.rs b/src/parser/actions/parse.rs index 648bfb53e..39e5207e8 100644 --- a/src/parser/actions/parse.rs +++ b/src/parser/actions/parse.rs @@ -241,7 +241,7 @@ pub fn parse( databases .txout_index_to_amount - .unsafe_insert(txout_index, amount); + .insert_to_ram(txout_index, amount); if compute_addresses { let address = address.unwrap(); @@ -253,7 +253,7 @@ pub fn parse( if let Some(address_index) = address_index_opt.or_else(|| { databases .address_to_address_index - .unsafe_get_from_puts(&address) + .get_from_ram(&address) .cloned() }) { let address_data = address_index_to_address_data @@ -301,7 +301,7 @@ pub fn parse( databases .txout_index_to_address_index - .unsafe_insert(txout_index, address_index); + .insert_to_ram(txout_index, address_index); } }); @@ -336,9 +336,7 @@ pub fn parse( .or_else(|| { is_tx_data_from_cached_puts = true; - databases - .txid_to_tx_data - .unsafe_get_mut_from_puts(&input_txid) + databases.txid_to_tx_data.get_mut_from_ram(&input_txid) }); // Can be none because 0 sats inputs happen @@ -534,7 +532,7 @@ pub fn parse( if remove_tx_data_from_cached_puts { // Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations) - databases.txid_to_tx_data.remove_from_puts(&input_txid) + databases.txid_to_tx_data.remove_from_ram(&input_txid) } ControlFlow::Continue(()) @@ -558,7 +556,7 @@ pub fn parse( txid_to_tx_data.into_iter().for_each(|(txid, tx_data)| { if let Some(tx_data) = tx_data { if tx_data.is_empty() { - databases.txid_to_tx_data.remove_from_db(txid); + databases.txid_to_tx_data.remove_later_from_disk(txid); } else { databases.txid_to_tx_data.update(txid, tx_data); } @@ -738,14 +736,14 @@ pub fn parse( address_index_to_address_data.unwrap().into_iter().for_each( |(address_index, address_data)| { if address_data.is_empty() { - databases.address_index_to_empty_address_data.unsafe_insert( + databases.address_index_to_empty_address_data.insert_to_ram( address_index, EmptyAddressData::from_non_empty(&address_data), ); } else { databases .address_index_to_address_data - .unsafe_insert(address_index, address_data); + .insert_to_ram(address_index, address_data); } }, ) @@ -901,7 +899,7 @@ fn prepare_inputs<'a>( let mut tx_datas = txid_to_tx_data .par_iter() - .map(|(txid, _)| txid_to_tx_data_db.unsafe_get(txid)) + .map(|(txid, _)| txid_to_tx_data_db.get(txid)) .collect::>(); txid_to_tx_data.values_mut().rev().for_each(|tx_data_opt| { @@ -992,20 +990,20 @@ fn compute_address_index_to_address_data( .par_iter_mut() .for_each(|(address_index, address_data)| { if let Some(_address_data) = - address_index_to_address_data_db.unsafe_get_from_cache(address_index) + address_index_to_address_data_db.get_from_ram(address_index) { _address_data.clone_into(address_data); } else if let Some(empty_address_data) = - address_index_to_empty_address_data_db.unsafe_get_from_cache(address_index) + address_index_to_empty_address_data_db.get_from_ram(address_index) { *address_data = AddressData::from_empty(empty_address_data); } else if let Some(_address_data) = - address_index_to_address_data_db.unsafe_get_from_db(address_index) + address_index_to_address_data_db.get_from_disk(address_index) { _address_data.clone_into(address_data); } else { let empty_address_data = address_index_to_empty_address_data_db - .unsafe_get_from_db(address_index) + .get_from_disk(address_index) .unwrap(); *address_data = AddressData::from_empty(empty_address_data); diff --git a/src/parser/databases/_database.rs b/src/parser/databases/_database.rs deleted file mode 100644 index bd61f1078..000000000 --- a/src/parser/databases/_database.rs +++ /dev/null @@ -1,229 +0,0 @@ -// https://docs.rs/sanakirja/latest/sanakirja/index.html -// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ - -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::Debug, - fs, io, mem, - path::PathBuf, -}; - -use allocative::Allocative; - -use sanakirja::{ - btree::{self, page, Db_, Iter}, - Commit, Env, Error, MutTxn, RootDb, Storable, -}; - -/// -/// Simple wrapper around Sanakirja Database with cached puts and dels for safe use outside exports. -/// -/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching "gets" along the way. -/// -#[derive(Allocative)] -#[allocative(bound = "Key: Allocative, Value: Allocative")] -pub struct Database -where - Key: Ord + Clone + Debug + Storable, - Value: Storable + PartialEq, -{ - path: PathBuf, - cached_puts: BTreeMap, - cached_dels: BTreeSet, - #[allocative(skip)] - db: Db_>, - #[allocative(skip)] - txn: MutTxn, -} - -const ROOT_DB: usize = 0; -const PAGE_SIZE: u64 = 4096; - -impl Database -where - Key: Ord + Clone + Debug + Storable, - Value: Storable + PartialEq, -{ - pub fn open(path: PathBuf) -> color_eyre::Result { - let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? }; - - let mut txn = Env::mut_txn_begin(env)?; - - let db = txn - .root_db(ROOT_DB) - .unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() }); - - Ok(Self { - path, - cached_puts: BTreeMap::default(), - cached_dels: BTreeSet::default(), - db, - txn, - }) - } - - #[inline] - pub fn iter(&self) -> Iter<'_, MutTxn, Key, Value, page::Page> { - btree::iter(&self.txn, &self.db, None).unwrap() - } - - pub fn collect(&self) -> BTreeMap - where - Value: Clone, - { - self.iter() - .map(|r| r.unwrap()) - .map(|(key, value)| (key.clone(), value.clone())) - .collect::<_>() - } - - #[inline] - pub fn get(&self, key: &Key) -> Option<&Value> { - if let Some(cached_put) = self.get_from_puts(key) { - return Some(cached_put); - } - - self.db_get(key) - } - - #[inline] - pub fn db_get(&self, key: &Key) -> Option<&Value> { - let option = btree::get(&self.txn, &self.db, key, None).unwrap(); - - if let Some((key_found, v)) = option { - if key == key_found { - return Some(v); - } - } - - None - } - - #[inline] - pub fn get_from_puts(&self, key: &Key) -> Option<&Value> { - self.cached_puts.get(key) - } - - #[inline] - pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> { - self.cached_puts.get_mut(key) - } - - #[inline] - pub fn remove(&mut self, key: &Key) -> Option { - self.remove_from_puts(key).or_else(|| { - self.db_remove(key); - - None - }) - } - - #[inline] - pub fn db_remove(&mut self, key: &Key) { - self.cached_dels.insert(key.clone()); - } - - #[inline] - pub fn update(&mut self, key: Key, value: Value) -> Option { - self.cached_dels.insert(key.clone()); - self.cached_puts.insert(key, value) - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.iter().next().is_none() - } - - #[inline] - pub fn remove_from_puts(&mut self, key: &Key) -> Option { - self.cached_puts.remove(key) - } - - #[inline] - pub fn insert(&mut self, key: Key, value: Value) -> Option { - self.cached_dels.remove(&key); - self.unsafe_insert(key, value) - } - - #[inline] - pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { - self.cached_puts.insert(key, value) - } - - fn db_multi_put(&mut self, tree: BTreeMap) -> Result<(), Error> { - tree.into_iter() - .try_for_each(|(key, value)| -> Result<(), Error> { - btree::put(&mut self.txn, &mut self.db, &key, &value)?; - Ok(()) - }) - } - - fn db_multi_del(&mut self, tree: BTreeSet) -> Result<(), Error> { - tree.into_iter().try_for_each(|key| -> Result<(), Error> { - btree::del(&mut self.txn, &mut self.db, &key, None)?; - Ok(()) - }) - } -} - -pub trait AnyDatabase { - #[allow(unused)] - fn export(self, defragment: bool) -> color_eyre::Result<(), Error>; - fn boxed_export(self: Box, defragment: bool) -> color_eyre::Result<(), Error>; - #[allow(unused)] - fn destroy(self) -> io::Result<()>; -} - -impl AnyDatabase for Database -where - Key: Ord + Clone + Debug + Storable, - Value: Storable + PartialEq + Clone, -{ - fn export(self, defragment: bool) -> color_eyre::Result<(), Error> { - Box::new(self).boxed_export(defragment) - } - - fn boxed_export(mut self: Box, defragment: bool) -> color_eyre::Result<(), Error> { - if defragment { - let mut btree = self.as_ref().collect(); - - let path = self.path.to_owned(); - self.cached_dels.iter().for_each(|key| { - btree.remove(key); - }); - btree.append(&mut self.cached_puts); - - self.destroy()?; - - *self = Self::open(path).unwrap(); - - if !self.is_empty() { - panic!() - } - - self.cached_puts = btree; - } - - if self.cached_dels.is_empty() && self.cached_puts.is_empty() { - return Ok(()); - } - - let cached_dels = mem::take(&mut self.cached_dels); - self.db_multi_del(cached_dels)?; - - let cached_puts = mem::take(&mut self.cached_puts); - self.db_multi_put(cached_puts)?; - - self.txn.set_root(ROOT_DB, self.db.db.into()); - - self.txn.commit() - } - - fn destroy(self) -> io::Result<()> { - let path = self.path.to_owned(); - - drop(self); - - fs::remove_file(&path) - } -} diff --git a/src/parser/databases/_trait.rs b/src/parser/databases/_trait.rs index f238c82b1..5da93dd8a 100644 --- a/src/parser/databases/_trait.rs +++ b/src/parser/databases/_trait.rs @@ -1,10 +1,11 @@ use std::{fs, io, path::Path}; use log::info; +use snkrj::AnyDatabase; use crate::structs::{Config, Date, Height}; -use super::{AnyDatabase, Metadata}; +use super::Metadata; pub trait AnyDatabaseGroup where diff --git a/src/parser/databases/address_index_to_address_data.rs b/src/parser/databases/address_index_to_address_data.rs index 82c90acf0..a56137b7f 100644 --- a/src/parser/databases/address_index_to_address_data.rs +++ b/src/parser/databases/address_index_to_address_data.rs @@ -8,6 +8,7 @@ use std::{ use allocative::Allocative; use itertools::Itertools; use rayon::prelude::*; +use snkrj::{AnyDatabase, Database as _Database}; use crate::{ parser::states::AddressCohortsDurableStates, @@ -15,7 +16,7 @@ use crate::{ utils::time, }; -use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata}; +use super::{AnyDatabaseGroup, Metadata}; type Key = u32; type Value = AddressData; @@ -25,6 +26,7 @@ type Database = _Database; pub struct AddressIndexToAddressData { path: PathBuf, pub metadata: Metadata, + #[allocative(skip)] pub map: BTreeMap, } @@ -45,10 +47,10 @@ impl DerefMut for AddressIndexToAddressData { pub const ADDRESS_INDEX_DB_MAX_SIZE: usize = 250_000; impl AddressIndexToAddressData { - pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { + pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option { self.metadata.called_insert(); - self.open_db(&key).unsafe_insert(key, value) + self.open_db(&key).insert_to_ram(key, value) } pub fn remove(&mut self, key: &Key) -> Option { @@ -59,16 +61,16 @@ impl AddressIndexToAddressData { /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed /// Though it makes it easy to use with rayon. - pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> { + pub fn get_from_ram(&self, key: &Key) -> Option<&Value> { let db_index = Self::db_index(key); - self.get(&db_index).unwrap().get_from_puts(key) + self.get(&db_index).unwrap().get_from_ram(key) } - pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> { + pub fn get_from_disk(&self, key: &Key) -> Option<&Value> { let db_index = Self::db_index(key); - self.get(&db_index).unwrap().db_get(key) + self.get(&db_index).unwrap().get_from_disk(key) } pub fn open_db(&mut self, key: &Key) -> &mut Database { @@ -99,7 +101,7 @@ impl AddressIndexToAddressData { let mut s = AddressCohortsDurableStates::default(); database - .iter() + .iter_disk() .map(|r| r.unwrap().1) .for_each(|address_data| s.increment(address_data).unwrap()); diff --git a/src/parser/databases/address_index_to_empty_address_data.rs b/src/parser/databases/address_index_to_empty_address_data.rs index 923ae4850..df940c2f3 100644 --- a/src/parser/databases/address_index_to_empty_address_data.rs +++ b/src/parser/databases/address_index_to_empty_address_data.rs @@ -7,12 +7,11 @@ use std::{ use allocative::Allocative; use itertools::Itertools; +use snkrj::{AnyDatabase, Database as _Database}; use crate::structs::{Config, EmptyAddressData}; -use super::{ - AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE, -}; +use super::{AnyDatabaseGroup, Metadata, ADDRESS_INDEX_DB_MAX_SIZE}; type Key = u32; type Value = EmptyAddressData; @@ -22,6 +21,7 @@ type Database = _Database; pub struct AddressIndexToEmptyAddressData { path: PathBuf, pub metadata: Metadata, + #[allocative(skip)] map: BTreeMap, } @@ -40,10 +40,10 @@ impl DerefMut for AddressIndexToEmptyAddressData { } impl AddressIndexToEmptyAddressData { - pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { + pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option { self.metadata.called_insert(); - self.open_db(&key).unsafe_insert(key, value) + self.open_db(&key).insert_to_ram(key, value) } pub fn remove(&mut self, key: &Key) -> Option { @@ -54,13 +54,13 @@ impl AddressIndexToEmptyAddressData { /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed /// Though it makes it easy to use with rayon. - pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> { + pub fn get_from_ram(&self, key: &Key) -> Option<&Value> { let db_index = Self::db_index(key); - self.get(&db_index).and_then(|db| db.get_from_puts(key)) + self.get(&db_index).and_then(|db| db.get_from_ram(key)) } - pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> { + pub fn get_from_disk(&self, key: &Key) -> Option<&Value> { let db_index = Self::db_index(key); self.get(&db_index) @@ -68,7 +68,7 @@ impl AddressIndexToEmptyAddressData { dbg!(&self.map.keys(), &key, &db_index); panic!() }) - .db_get(key) + .get_from_disk(key) } pub fn open_db(&mut self, key: &Key) -> &mut Database { diff --git a/src/parser/databases/address_to_address_index.rs b/src/parser/databases/address_to_address_index.rs index 35d9b7e30..01518a9fe 100644 --- a/src/parser/databases/address_to_address_index.rs +++ b/src/parser/databases/address_to_address_index.rs @@ -6,10 +6,11 @@ use std::{ use allocative::Allocative; use itertools::Itertools; +use snkrj::{AnyDatabase, Database}; use crate::structs::{Address, Config, U8x19, U8x31}; -use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata}; +use super::{AnyDatabaseGroup, Metadata}; type Value = u32; type U8x19Database = Database; @@ -33,16 +34,27 @@ pub struct AddressToAddressIndex { path: PathBuf, pub metadata: Metadata, + #[allocative(skip)] p2pk: BTreeMap, + #[allocative(skip)] p2pkh: BTreeMap, + #[allocative(skip)] p2sh: BTreeMap, + #[allocative(skip)] p2wpkh: BTreeMap, + #[allocative(skip)] p2wsh: BTreeMap, + #[allocative(skip)] p2tr: BTreeMap, + #[allocative(skip)] op_return: Option, + #[allocative(skip)] push_only: Option, + #[allocative(skip)] unknown: Option, + #[allocative(skip)] empty: Option, + #[allocative(skip)] multisig: Option, } @@ -103,19 +115,19 @@ impl AddressToAddressIndex { } } - pub fn unsafe_get_from_puts(&self, address: &Address) -> Option<&Value> { + pub fn get_from_ram(&self, address: &Address) -> Option<&Value> { match address { - Address::Empty(key) => self.empty.as_ref().unwrap().get_from_puts(key), - Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_puts(key), - Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_puts(key), - Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_puts(key), - Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_puts(key), - Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_puts(key), - Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_puts(key), - Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_puts(key), - Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_puts(key), - Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_puts(key), - Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_puts(key), + Address::Empty(key) => self.empty.as_ref().unwrap().get_from_ram(key), + Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_ram(key), + Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_ram(key), + Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_ram(key), + Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_ram(key), + Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_ram(key), + Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_ram(key), + Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_ram(key), + Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_ram(key), + Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_ram(key), + Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_ram(key), } } diff --git a/src/parser/databases/mod.rs b/src/parser/databases/mod.rs index 9b32a982c..3370d914a 100644 --- a/src/parser/databases/mod.rs +++ b/src/parser/databases/mod.rs @@ -1,6 +1,5 @@ use allocative::Allocative; -mod _database; mod _trait; mod address_index_to_address_data; mod address_index_to_empty_address_data; @@ -10,7 +9,6 @@ mod txid_to_tx_data; mod txout_index_to_address_index; mod txout_index_to_amount; -pub use _database::*; use _trait::*; pub use address_index_to_address_data::*; pub use address_index_to_empty_address_data::*; @@ -19,6 +17,7 @@ use itertools::Itertools; use log::info; use metadata::*; use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use snkrj::AnyDatabase; pub use txid_to_tx_data::*; pub use txout_index_to_address_index::*; pub use txout_index_to_amount::*; diff --git a/src/parser/databases/txid_to_tx_data.rs b/src/parser/databases/txid_to_tx_data.rs index 14ee58555..399c760d9 100644 --- a/src/parser/databases/txid_to_tx_data.rs +++ b/src/parser/databases/txid_to_tx_data.rs @@ -1,17 +1,17 @@ use std::{ collections::BTreeMap, fs, mem, - ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; use allocative::Allocative; use biter::bitcoin::Txid; use itertools::Itertools; +use snkrj::{AnyDatabase, Database as _Database}; use crate::structs::{Config, TxData, U8x31}; -use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata}; +use super::{AnyDatabaseGroup, Metadata}; type Key = U8x31; type Value = TxData; @@ -21,23 +21,10 @@ type Database = _Database; pub struct TxidToTxData { path: PathBuf, pub metadata: Metadata, + #[allocative(skip)] map: BTreeMap, } -impl Deref for TxidToTxData { - type Target = BTreeMap; - - fn deref(&self) -> &Self::Target { - &self.map - } -} - -impl DerefMut for TxidToTxData { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.map - } -} - impl TxidToTxData { pub fn insert(&mut self, txid: &Txid, tx_index: Value) -> Option { self.metadata.called_insert(); @@ -47,53 +34,41 @@ impl TxidToTxData { self.open_db(txid).insert(txid_key, tx_index) } - // pub fn safe_get(&mut self, txid: &Txid) -> Option<&Value> { - // let txid_key = Self::txid_to_key(txid); - // self.open_db(txid).get(&txid_key) - // } - /// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed. /// Though it makes it easy to use with rayon - pub fn unsafe_get(&self, txid: &Txid) -> Option<&Value> { + pub fn get(&self, txid: &Txid) -> Option<&Value> { let txid_key = Self::txid_to_key(txid); let db_index = Self::db_index(txid); - self.get(&db_index).unwrap().get(&txid_key) + self.map.get(&db_index).unwrap().get(&txid_key) } - // pub fn unsafe_get_from_puts(&self, txid: &Txid) -> Option<&Value> { - // let txid_key = Self::txid_to_key(txid); - - // let db_index = Self::db_index(txid); - - // self.get(&db_index).unwrap().get_from_puts(&txid_key) - // } - - pub fn unsafe_get_mut_from_puts(&mut self, txid: &Txid) -> Option<&mut Value> { + pub fn get_mut_from_ram(&mut self, txid: &Txid) -> Option<&mut Value> { let txid_key = Self::txid_to_key(txid); let db_index = Self::db_index(txid); - self.get_mut(&db_index) + self.map + .get_mut(&db_index) .unwrap() - .get_mut_from_puts(&txid_key) + .get_mut_from_ram(&txid_key) } - pub fn remove_from_db(&mut self, txid: &Txid) { + pub fn remove_later_from_disk(&mut self, txid: &Txid) { self.metadata.called_remove(); let txid_key = Self::txid_to_key(txid); - self.open_db(txid).db_remove(&txid_key); + self.open_db(txid).remove_later_from_disk(&txid_key); } - pub fn remove_from_puts(&mut self, txid: &Txid) { + pub fn remove_from_ram(&mut self, txid: &Txid) { self.metadata.called_remove(); let txid_key = Self::txid_to_key(txid); - self.open_db(txid).remove_from_puts(&txid_key); + self.open_db(txid).remove_from_ram(&txid_key); } pub fn update(&mut self, txid: &Txid, tx_data: TxData) { @@ -111,7 +86,7 @@ impl TxidToTxData { #[inline(always)] pub fn _open_db(&mut self, db_index: u16) -> &mut Database { let path = self.path.to_owned(); - self.entry(db_index).or_insert_with(|| { + self.map.entry(db_index).or_insert_with(|| { let path = path.join(db_index.to_string()); Database::open(path).unwrap() }) diff --git a/src/parser/databases/txout_index_to_address_index.rs b/src/parser/databases/txout_index_to_address_index.rs index 5db9f4484..4e060ddfc 100644 --- a/src/parser/databases/txout_index_to_address_index.rs +++ b/src/parser/databases/txout_index_to_address_index.rs @@ -7,10 +7,11 @@ use std::{ use allocative::Allocative; use itertools::Itertools; +use snkrj::{AnyDatabase, Database as _Database}; use crate::structs::{Config, TxoutIndex}; -use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata}; +use super::{AnyDatabaseGroup, Metadata}; type Key = TxoutIndex; type Value = u32; @@ -20,6 +21,7 @@ type Database = _Database; pub struct TxoutIndexToAddressIndex { path: PathBuf, pub metadata: Metadata, + #[allocative(skip)] map: BTreeMap, } @@ -40,20 +42,12 @@ impl DerefMut for TxoutIndexToAddressIndex { const DB_MAX_SIZE: usize = 10_000_000_000; impl TxoutIndexToAddressIndex { - pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { + pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option { self.metadata.called_insert(); - self.open_db(&key).unsafe_insert(key, value) + self.open_db(&key).insert_to_ram(key, value) } - // pub fn undo_insert(&mut self, key: &Key) -> Option { - // self.open_db(key).remove_from_puts(key).map(|v| { - // self.metadata.called_remove(); - - // v - // }) - // } - pub fn remove(&mut self, key: &Key) -> Option { self.metadata.called_remove(); diff --git a/src/parser/databases/txout_index_to_amount.rs b/src/parser/databases/txout_index_to_amount.rs index 0464b04c7..bc86a2c2e 100644 --- a/src/parser/databases/txout_index_to_amount.rs +++ b/src/parser/databases/txout_index_to_amount.rs @@ -7,10 +7,11 @@ use std::{ use allocative::Allocative; use itertools::Itertools; +use snkrj::{AnyDatabase, Database as _Database}; use crate::structs::{Amount, Config, TxoutIndex}; -use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata}; +use super::{AnyDatabaseGroup, Metadata}; type Key = TxoutIndex; type Value = Amount; @@ -20,6 +21,7 @@ type Database = _Database; pub struct TxoutIndexToAmount { path: PathBuf, pub metadata: Metadata, + #[allocative(skip)] map: BTreeMap, } @@ -40,20 +42,12 @@ impl DerefMut for TxoutIndexToAmount { const DB_MAX_SIZE: usize = 10_000_000_000; impl TxoutIndexToAmount { - pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { + pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option { self.metadata.called_insert(); - self.open_db(&key).unsafe_insert(key, value) + self.open_db(&key).insert_to_ram(key, value) } - // pub fn undo_insert(&mut self, key: &Key) -> Option { - // self.open_db(key).remove_from_puts(key).map(|v| { - // self.metadata.called_remove(); - - // v - // }) - // } - pub fn remove(&mut self, key: &Key) -> Option { self.metadata.called_remove(); diff --git a/src/structs/address_data.rs b/src/structs/address_data.rs index 3d0968d0a..636819317 100644 --- a/src/structs/address_data.rs +++ b/src/structs/address_data.rs @@ -1,6 +1,6 @@ use allocative::Allocative; use color_eyre::eyre::eyre; -use sanakirja::{direct_repr, Storable, UnsizedStorable}; +use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::{AddressType, Amount, EmptyAddressData, LiquidityClassification, Price}; diff --git a/src/structs/amount.rs b/src/structs/amount.rs index 6ff8ce1d4..8308a7d40 100644 --- a/src/structs/amount.rs +++ b/src/structs/amount.rs @@ -12,8 +12,8 @@ use bincode::{ }; use biter::bitcoin::Amount as BitcoinAmount; use derive_deref::{Deref, DerefMut}; -use sanakirja::{direct_repr, Storable, UnsizedStorable}; use serde::{Deserialize, Serialize}; +use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::Height; diff --git a/src/structs/array.rs b/src/structs/array.rs index dbfb67592..8a915db6d 100644 --- a/src/structs/array.rs +++ b/src/structs/array.rs @@ -2,7 +2,7 @@ use std::fmt::Debug; use allocative::Allocative; use derive_deref::{Deref, DerefMut}; -use sanakirja::{direct_repr, Storable, UnsizedStorable}; +use snkrj::{direct_repr, Storable, UnsizedStorable}; #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative, diff --git a/src/structs/date.rs b/src/structs/date.rs index c2d51dd51..da448f6bb 100644 --- a/src/structs/date.rs +++ b/src/structs/date.rs @@ -74,6 +74,10 @@ impl Date { self.month() == 1 } + pub fn is_june(&self) -> bool { + self.month() == 6 + } + pub fn is_first_of_month(&self) -> bool { self.day() == 1 } diff --git a/src/structs/empty_address_data.rs b/src/structs/empty_address_data.rs index a7613efcb..48aaa7784 100644 --- a/src/structs/empty_address_data.rs +++ b/src/structs/empty_address_data.rs @@ -1,5 +1,5 @@ use allocative::Allocative; -use sanakirja::{direct_repr, Storable, UnsizedStorable}; +use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::{AddressData, AddressType, Amount}; diff --git a/src/structs/tx_data.rs b/src/structs/tx_data.rs index 1a577e8be..162ed456f 100644 --- a/src/structs/tx_data.rs +++ b/src/structs/tx_data.rs @@ -1,5 +1,5 @@ use allocative::Allocative; -use sanakirja::{direct_repr, Storable, UnsizedStorable}; +use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::BlockPath; diff --git a/src/structs/txout_index.rs b/src/structs/txout_index.rs index 6da7f588d..ebb36775f 100644 --- a/src/structs/txout_index.rs +++ b/src/structs/txout_index.rs @@ -1,6 +1,6 @@ use allocative::Allocative; use bincode::{Decode, Encode}; -use sanakirja::{direct_repr, Storable, UnsizedStorable}; +use snkrj::{direct_repr, Storable, UnsizedStorable}; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Encode, Decode, Allocative)] pub struct TxoutIndex {