mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-05-29 21:39:26 -07:00
snkrj: move database struct to its own crate
This commit is contained in:
nym21
@@ -6,11 +6,12 @@
|
||||
# v0.6.0 | WIP
|
||||
|
||||
- Merged parser and server crates into a single project (and thus executable)
|
||||
- Started using `log` and `env_logger` crates
|
||||
- Started using `log` and `env_logger` crates instead of custom code
|
||||
- Improved logs
|
||||
- Added `--server BOOL` and `--parser BOOL` parameters (both are true by default)
|
||||
- Automated databases defragmention (and removed parameter)
|
||||
- Fixed input being unfocused right after being focused in Brave browser
|
||||
- Moved Sanakirja database wrapper to its own crate: `snkrj`
|
||||
|
||||
# [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
|
||||
|
||||
9
Cargo.lock
generated
9
Cargo.lock
generated
@@ -1524,9 +1524,9 @@ dependencies = [
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rlimit",
|
||||
"sanakirja",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snkrj",
|
||||
"struct_iterable",
|
||||
"swc",
|
||||
"swc_common",
|
||||
@@ -2621,6 +2621,13 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snkrj"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"sanakirja",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.5.7"
|
||||
|
||||
@@ -27,7 +27,7 @@ rayon = "1.10.0"
|
||||
regex = "1.11.1"
|
||||
reqwest = { version = "0.12.9", features = ["blocking", "json"] }
|
||||
rlimit = "0.10.2"
|
||||
sanakirja = "1.4.3"
|
||||
snkrj = { path = "./crates/snkrj" }
|
||||
serde = { version = "1.0.216", features = ["derive"] }
|
||||
serde_json = "1.0.133"
|
||||
struct_iterable = { path = "./crates/iterable" }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Biter
|
||||
Copyright (c) 2024 biter
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
242
crates/snkrj/Cargo.lock
generated
Normal file
242
crates/snkrj/Cargo.lock
generated
Normal file
@@ -0,0 +1,242 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.168"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sanakirja"
|
||||
version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81aaf70d064e2122209f04d01fd91e8908e7a327b516236e1cbc0c3f34ac6d11"
|
||||
dependencies = [
|
||||
"fs2",
|
||||
"log",
|
||||
"memmap2",
|
||||
"parking_lot",
|
||||
"sanakirja-core",
|
||||
"serde",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sanakirja-core"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.216"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.216"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
|
||||
|
||||
[[package]]
|
||||
name = "snkrj"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"sanakirja",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.90"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
12
crates/snkrj/Cargo.toml
Normal file
12
crates/snkrj/Cargo.toml
Normal file
@@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "snkrj"
|
||||
description = "A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap"
|
||||
version = "0.1.0"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/kibo-money/kibo/tree/main/crates/snkrj"
|
||||
keywords = ["database", "sanakirja", "btreemap"]
|
||||
categories = ["database"]
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
sanakirja = "1.4.3"
|
||||
21
crates/snkrj/LICENSE.md
Normal file
21
crates/snkrj/LICENSE.md
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 snkrj
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
37
crates/snkrj/README.md
Normal file
37
crates/snkrj/README.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# snkrj
|
||||
|
||||
A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap.
|
||||
|
||||
## Example
|
||||
|
||||
```rust
|
||||
use snkrj::{AnyDatabase, Database};
|
||||
|
||||
fn main() {
|
||||
let path = std::env::temp_dir().join("./db");
|
||||
|
||||
let database: Database<i32, i32> = Database::open(path.clone()).unwrap();
|
||||
let _ = database.destroy();
|
||||
|
||||
let mut database: Database<i32, i32> = Database::open(path.clone()).unwrap();
|
||||
database.insert(64, 128);
|
||||
database.export(false).unwrap();
|
||||
|
||||
let mut database: Database<i32, i32> = Database::open(path).unwrap();
|
||||
database.insert(1, 2);
|
||||
database.insert(128, 256);
|
||||
println!("iter_ram:");
|
||||
database.iter_ram().for_each(|pair| {
|
||||
println!("{:?}", pair);
|
||||
});
|
||||
println!("iter_disk:");
|
||||
database.iter_disk().for_each(|pair| {
|
||||
println!("{:?}", pair.unwrap());
|
||||
});
|
||||
println!("iter_ram_then_disk:");
|
||||
database.iter_ram_then_disk().for_each(|pair| {
|
||||
println!("{:?}", pair);
|
||||
});
|
||||
database.export(false).unwrap();
|
||||
}
|
||||
```
|
||||
252
crates/snkrj/src/lib.rs
Normal file
252
crates/snkrj/src/lib.rs
Normal file
@@ -0,0 +1,252 @@
|
||||
// https://docs.rs/sanakirja/latest/sanakirja/index.html
|
||||
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt::Debug,
|
||||
fs, io, mem,
|
||||
path::PathBuf,
|
||||
result::Result,
|
||||
};
|
||||
|
||||
use sanakirja::btree::{page, Db_};
|
||||
pub use sanakirja::*;
|
||||
|
||||
///
|
||||
/// A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap.
|
||||
///
|
||||
/// The state of the tree is uncommited until `.export()` is called during which it is unsafe to stop the program.
|
||||
///
|
||||
pub struct Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq,
|
||||
{
|
||||
path: PathBuf,
|
||||
puts: BTreeMap<Key, Value>,
|
||||
dels: BTreeSet<Key>,
|
||||
db: Db_<Key, Value, page::Page<Key, Value>>,
|
||||
txn: MutTxn<Env, ()>,
|
||||
}
|
||||
|
||||
const ROOT_DB: usize = 0;
|
||||
const PAGE_SIZE: u64 = 4096;
|
||||
|
||||
impl<Key, Value> Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq,
|
||||
{
|
||||
/// Open a database without a lock file where only one instance is safe to open.
|
||||
pub fn open(path: PathBuf) -> Result<Self, Error> {
|
||||
let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? };
|
||||
|
||||
let mut txn = Env::mut_txn_begin(env)?;
|
||||
|
||||
let db = txn
|
||||
.root_db(ROOT_DB)
|
||||
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
|
||||
|
||||
Ok(Self {
|
||||
path,
|
||||
puts: BTreeMap::default(),
|
||||
dels: BTreeSet::default(),
|
||||
db,
|
||||
txn,
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get(&self, key: &Key) -> Option<&Value> {
|
||||
if let Some(cached_put) = self.get_from_ram(key) {
|
||||
return Some(cached_put);
|
||||
}
|
||||
|
||||
self.get_from_disk(key)
|
||||
}
|
||||
|
||||
/// Get only from the uncommited tree (ram) without checking the database (disk)
|
||||
#[inline]
|
||||
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
|
||||
self.puts.get(key)
|
||||
}
|
||||
|
||||
/// Get mut only from the uncommited tree (ram) without checking the database (disk)
|
||||
#[inline]
|
||||
pub fn get_mut_from_ram(&mut self, key: &Key) -> Option<&mut Value> {
|
||||
self.puts.get_mut(key)
|
||||
}
|
||||
|
||||
/// Get only from the database (disk) without checking the uncommited tree (ram)
|
||||
#[inline]
|
||||
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
|
||||
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
|
||||
|
||||
if let Some((key_found, v)) = option {
|
||||
if key == key_found {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.dels.remove(&key);
|
||||
self.insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
/// Insert without removing the key to the dels tree, so be sure that it hasn't added to the delete set
|
||||
#[inline]
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.puts.insert(key, value)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.dels.insert(key.clone());
|
||||
self.puts.insert(key, value)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.remove_from_ram(key).or_else(|| {
|
||||
self.remove_later_from_disk(key);
|
||||
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
/// Get only from the uncommited tree (ram) without checking the database (disk)
|
||||
#[inline]
|
||||
pub fn remove_from_ram(&mut self, key: &Key) -> Option<Value> {
|
||||
self.puts.remove(key)
|
||||
}
|
||||
|
||||
/// Add the key only to the dels tree without checking if it's present in the puts tree, only use if you are positive that you neither added nor updated an entry with this key
|
||||
#[inline]
|
||||
pub fn remove_later_from_disk(&mut self, key: &Key) {
|
||||
self.dels.insert(key.clone());
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.iter_disk().next().is_none()
|
||||
}
|
||||
|
||||
/// Iterate over key/value pairs from the uncommited tree (ram)
|
||||
#[inline]
|
||||
pub fn iter_ram(&self) -> std::collections::btree_map::Iter<'_, Key, Value> {
|
||||
self.puts.iter()
|
||||
}
|
||||
|
||||
/// Iterate over key/value pairs from the database (disk)
|
||||
#[inline]
|
||||
pub fn iter_disk(
|
||||
&self,
|
||||
) -> btree::Iter<'_, MutTxn<Env, ()>, Key, Value, page::Page<Key, Value>> {
|
||||
btree::iter(&self.txn, &self.db, None).unwrap()
|
||||
}
|
||||
|
||||
/// Iterate over key/value pairs
|
||||
#[inline]
|
||||
pub fn iter_ram_then_disk(&self) -> impl Iterator<Item = (&Key, &Value)> {
|
||||
self.iter_ram().chain(self.iter_disk().map(|r| r.unwrap()))
|
||||
}
|
||||
|
||||
/// Collect a **clone** of all uncommited key/value pairs (ram)
|
||||
pub fn collect_ram(&self) -> BTreeMap<Key, Value>
|
||||
where
|
||||
Value: Clone,
|
||||
{
|
||||
self.puts.clone()
|
||||
}
|
||||
|
||||
/// Collect a **clone** of all key/value pairs from the database (disk)
|
||||
pub fn collect_disk(&self) -> BTreeMap<Key, Value>
|
||||
where
|
||||
Value: Clone,
|
||||
{
|
||||
self.iter_disk()
|
||||
.map(|r| r.unwrap())
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect::<_>()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyDatabase {
|
||||
#[allow(unused)]
|
||||
fn export(self, defragment: bool) -> Result<(), Error>;
|
||||
fn boxed_export(self: Box<Self>, defragment: bool) -> Result<(), Error>;
|
||||
#[allow(unused)]
|
||||
fn destroy(self) -> io::Result<()>;
|
||||
}
|
||||
|
||||
impl<Key, Value> AnyDatabase for Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq + Clone,
|
||||
{
|
||||
/// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space
|
||||
///
|
||||
/// /!\ Do not kill the program while this function is runnning /!\
|
||||
fn export(self, defragment: bool) -> Result<(), Error> {
|
||||
Box::new(self).boxed_export(defragment)
|
||||
}
|
||||
|
||||
/// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space
|
||||
///
|
||||
/// /!\ Do not kill the program while this function is runnning /!\
|
||||
fn boxed_export(mut self: Box<Self>, defragment: bool) -> Result<(), Error> {
|
||||
if defragment {
|
||||
let mut btree = self.as_ref().collect_disk();
|
||||
|
||||
let path = self.path.to_owned();
|
||||
self.dels.iter().for_each(|key| {
|
||||
btree.remove(key);
|
||||
});
|
||||
btree.append(&mut self.puts);
|
||||
|
||||
self.destroy()?;
|
||||
|
||||
*self = Self::open(path).unwrap();
|
||||
|
||||
if !self.is_empty() {
|
||||
panic!()
|
||||
}
|
||||
|
||||
self.puts = btree;
|
||||
}
|
||||
|
||||
if self.dels.is_empty() && self.puts.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
mem::take(&mut self.dels)
|
||||
.into_iter()
|
||||
.try_for_each(|key| -> Result<(), Error> {
|
||||
btree::del(&mut self.txn, &mut self.db, &key, None)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
mem::take(&mut self.puts).into_iter().try_for_each(
|
||||
|(key, value)| -> Result<(), Error> {
|
||||
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
||||
self.txn.set_root(ROOT_DB, self.db.db.into());
|
||||
|
||||
self.txn.commit()
|
||||
}
|
||||
|
||||
fn destroy(self) -> io::Result<()> {
|
||||
let path = self.path.to_owned();
|
||||
|
||||
drop(self);
|
||||
|
||||
fs::remove_file(&path)
|
||||
}
|
||||
}
|
||||
29
crates/snkrj/src/main.rs
Normal file
29
crates/snkrj/src/main.rs
Normal file
@@ -0,0 +1,29 @@
|
||||
use snkrj::{AnyDatabase, Database};
|
||||
|
||||
fn main() {
|
||||
let path = std::env::temp_dir().join("./db");
|
||||
|
||||
let database: Database<i32, i32> = Database::open(path.clone()).unwrap();
|
||||
let _ = database.destroy();
|
||||
|
||||
let mut database: Database<i32, i32> = Database::open(path.clone()).unwrap();
|
||||
database.insert(64, 128);
|
||||
database.export(false).unwrap();
|
||||
|
||||
let mut database: Database<i32, i32> = Database::open(path).unwrap();
|
||||
database.insert(1, 2);
|
||||
database.insert(128, 256);
|
||||
println!("iter_ram:");
|
||||
database.iter_ram().for_each(|pair| {
|
||||
println!("{:?}", pair);
|
||||
});
|
||||
println!("iter_disk:");
|
||||
database.iter_disk().for_each(|pair| {
|
||||
println!("{:?}", pair.unwrap());
|
||||
});
|
||||
println!("iter_ram_then_disk:");
|
||||
database.iter_ram_then_disk().for_each(|pair| {
|
||||
println!("{:?}", pair);
|
||||
});
|
||||
database.export(false).unwrap();
|
||||
}
|
||||
@@ -213,7 +213,9 @@ pub fn iter_blocks(
|
||||
|
||||
let defragment = is_safe
|
||||
&& next_date_opt.is_some_and(|date| {
|
||||
date.year() >= 2020 && date.is_january() && date.is_first_of_month()
|
||||
(date.year() >= 2020 && date.is_january()
|
||||
|| date.year() >= 2022 && date.is_june())
|
||||
&& date.is_first_of_month()
|
||||
});
|
||||
|
||||
export(ExportedData {
|
||||
|
||||
@@ -241,7 +241,7 @@ pub fn parse(
|
||||
|
||||
databases
|
||||
.txout_index_to_amount
|
||||
.unsafe_insert(txout_index, amount);
|
||||
.insert_to_ram(txout_index, amount);
|
||||
|
||||
if compute_addresses {
|
||||
let address = address.unwrap();
|
||||
@@ -253,7 +253,7 @@ pub fn parse(
|
||||
if let Some(address_index) = address_index_opt.or_else(|| {
|
||||
databases
|
||||
.address_to_address_index
|
||||
.unsafe_get_from_puts(&address)
|
||||
.get_from_ram(&address)
|
||||
.cloned()
|
||||
}) {
|
||||
let address_data = address_index_to_address_data
|
||||
@@ -301,7 +301,7 @@ pub fn parse(
|
||||
|
||||
databases
|
||||
.txout_index_to_address_index
|
||||
.unsafe_insert(txout_index, address_index);
|
||||
.insert_to_ram(txout_index, address_index);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -336,9 +336,7 @@ pub fn parse(
|
||||
.or_else(|| {
|
||||
is_tx_data_from_cached_puts = true;
|
||||
|
||||
databases
|
||||
.txid_to_tx_data
|
||||
.unsafe_get_mut_from_puts(&input_txid)
|
||||
databases.txid_to_tx_data.get_mut_from_ram(&input_txid)
|
||||
});
|
||||
|
||||
// Can be none because 0 sats inputs happen
|
||||
@@ -534,7 +532,7 @@ pub fn parse(
|
||||
|
||||
if remove_tx_data_from_cached_puts {
|
||||
// Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
|
||||
databases.txid_to_tx_data.remove_from_puts(&input_txid)
|
||||
databases.txid_to_tx_data.remove_from_ram(&input_txid)
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
@@ -558,7 +556,7 @@ pub fn parse(
|
||||
txid_to_tx_data.into_iter().for_each(|(txid, tx_data)| {
|
||||
if let Some(tx_data) = tx_data {
|
||||
if tx_data.is_empty() {
|
||||
databases.txid_to_tx_data.remove_from_db(txid);
|
||||
databases.txid_to_tx_data.remove_later_from_disk(txid);
|
||||
} else {
|
||||
databases.txid_to_tx_data.update(txid, tx_data);
|
||||
}
|
||||
@@ -738,14 +736,14 @@ pub fn parse(
|
||||
address_index_to_address_data.unwrap().into_iter().for_each(
|
||||
|(address_index, address_data)| {
|
||||
if address_data.is_empty() {
|
||||
databases.address_index_to_empty_address_data.unsafe_insert(
|
||||
databases.address_index_to_empty_address_data.insert_to_ram(
|
||||
address_index,
|
||||
EmptyAddressData::from_non_empty(&address_data),
|
||||
);
|
||||
} else {
|
||||
databases
|
||||
.address_index_to_address_data
|
||||
.unsafe_insert(address_index, address_data);
|
||||
.insert_to_ram(address_index, address_data);
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -901,7 +899,7 @@ fn prepare_inputs<'a>(
|
||||
|
||||
let mut tx_datas = txid_to_tx_data
|
||||
.par_iter()
|
||||
.map(|(txid, _)| txid_to_tx_data_db.unsafe_get(txid))
|
||||
.map(|(txid, _)| txid_to_tx_data_db.get(txid))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
txid_to_tx_data.values_mut().rev().for_each(|tx_data_opt| {
|
||||
@@ -992,20 +990,20 @@ fn compute_address_index_to_address_data(
|
||||
.par_iter_mut()
|
||||
.for_each(|(address_index, address_data)| {
|
||||
if let Some(_address_data) =
|
||||
address_index_to_address_data_db.unsafe_get_from_cache(address_index)
|
||||
address_index_to_address_data_db.get_from_ram(address_index)
|
||||
{
|
||||
_address_data.clone_into(address_data);
|
||||
} else if let Some(empty_address_data) =
|
||||
address_index_to_empty_address_data_db.unsafe_get_from_cache(address_index)
|
||||
address_index_to_empty_address_data_db.get_from_ram(address_index)
|
||||
{
|
||||
*address_data = AddressData::from_empty(empty_address_data);
|
||||
} else if let Some(_address_data) =
|
||||
address_index_to_address_data_db.unsafe_get_from_db(address_index)
|
||||
address_index_to_address_data_db.get_from_disk(address_index)
|
||||
{
|
||||
_address_data.clone_into(address_data);
|
||||
} else {
|
||||
let empty_address_data = address_index_to_empty_address_data_db
|
||||
.unsafe_get_from_db(address_index)
|
||||
.get_from_disk(address_index)
|
||||
.unwrap();
|
||||
|
||||
*address_data = AddressData::from_empty(empty_address_data);
|
||||
|
||||
@@ -1,229 +0,0 @@
|
||||
// https://docs.rs/sanakirja/latest/sanakirja/index.html
|
||||
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt::Debug,
|
||||
fs, io, mem,
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
|
||||
use sanakirja::{
|
||||
btree::{self, page, Db_, Iter},
|
||||
Commit, Env, Error, MutTxn, RootDb, Storable,
|
||||
};
|
||||
|
||||
///
|
||||
/// Simple wrapper around Sanakirja Database with cached puts and dels for safe use outside exports.
|
||||
///
|
||||
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching "gets" along the way.
|
||||
///
|
||||
#[derive(Allocative)]
|
||||
#[allocative(bound = "Key: Allocative, Value: Allocative")]
|
||||
pub struct Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq,
|
||||
{
|
||||
path: PathBuf,
|
||||
cached_puts: BTreeMap<Key, Value>,
|
||||
cached_dels: BTreeSet<Key>,
|
||||
#[allocative(skip)]
|
||||
db: Db_<Key, Value, page::Page<Key, Value>>,
|
||||
#[allocative(skip)]
|
||||
txn: MutTxn<Env, ()>,
|
||||
}
|
||||
|
||||
const ROOT_DB: usize = 0;
|
||||
const PAGE_SIZE: u64 = 4096;
|
||||
|
||||
impl<Key, Value> Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq,
|
||||
{
|
||||
pub fn open(path: PathBuf) -> color_eyre::Result<Self> {
|
||||
let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? };
|
||||
|
||||
let mut txn = Env::mut_txn_begin(env)?;
|
||||
|
||||
let db = txn
|
||||
.root_db(ROOT_DB)
|
||||
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
|
||||
|
||||
Ok(Self {
|
||||
path,
|
||||
cached_puts: BTreeMap::default(),
|
||||
cached_dels: BTreeSet::default(),
|
||||
db,
|
||||
txn,
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn iter(&self) -> Iter<'_, MutTxn<Env, ()>, Key, Value, page::Page<Key, Value>> {
|
||||
btree::iter(&self.txn, &self.db, None).unwrap()
|
||||
}
|
||||
|
||||
pub fn collect(&self) -> BTreeMap<Key, Value>
|
||||
where
|
||||
Value: Clone,
|
||||
{
|
||||
self.iter()
|
||||
.map(|r| r.unwrap())
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect::<_>()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get(&self, key: &Key) -> Option<&Value> {
|
||||
if let Some(cached_put) = self.get_from_puts(key) {
|
||||
return Some(cached_put);
|
||||
}
|
||||
|
||||
self.db_get(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn db_get(&self, key: &Key) -> Option<&Value> {
|
||||
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
|
||||
|
||||
if let Some((key_found, v)) = option {
|
||||
if key == key_found {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_from_puts(&self, key: &Key) -> Option<&Value> {
|
||||
self.cached_puts.get(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> {
|
||||
self.cached_puts.get_mut(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.remove_from_puts(key).or_else(|| {
|
||||
self.db_remove(key);
|
||||
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn db_remove(&mut self, key: &Key) {
|
||||
self.cached_dels.insert(key.clone());
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.cached_dels.insert(key.clone());
|
||||
self.cached_puts.insert(key, value)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.iter().next().is_none()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn remove_from_puts(&mut self, key: &Key) -> Option<Value> {
|
||||
self.cached_puts.remove(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.cached_dels.remove(&key);
|
||||
self.unsafe_insert(key, value)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.cached_puts.insert(key, value)
|
||||
}
|
||||
|
||||
fn db_multi_put(&mut self, tree: BTreeMap<Key, Value>) -> Result<(), Error> {
|
||||
tree.into_iter()
|
||||
.try_for_each(|(key, value)| -> Result<(), Error> {
|
||||
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn db_multi_del(&mut self, tree: BTreeSet<Key>) -> Result<(), Error> {
|
||||
tree.into_iter().try_for_each(|key| -> Result<(), Error> {
|
||||
btree::del(&mut self.txn, &mut self.db, &key, None)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyDatabase {
|
||||
#[allow(unused)]
|
||||
fn export(self, defragment: bool) -> color_eyre::Result<(), Error>;
|
||||
fn boxed_export(self: Box<Self>, defragment: bool) -> color_eyre::Result<(), Error>;
|
||||
#[allow(unused)]
|
||||
fn destroy(self) -> io::Result<()>;
|
||||
}
|
||||
|
||||
impl<Key, Value> AnyDatabase for Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq + Clone,
|
||||
{
|
||||
fn export(self, defragment: bool) -> color_eyre::Result<(), Error> {
|
||||
Box::new(self).boxed_export(defragment)
|
||||
}
|
||||
|
||||
fn boxed_export(mut self: Box<Self>, defragment: bool) -> color_eyre::Result<(), Error> {
|
||||
if defragment {
|
||||
let mut btree = self.as_ref().collect();
|
||||
|
||||
let path = self.path.to_owned();
|
||||
self.cached_dels.iter().for_each(|key| {
|
||||
btree.remove(key);
|
||||
});
|
||||
btree.append(&mut self.cached_puts);
|
||||
|
||||
self.destroy()?;
|
||||
|
||||
*self = Self::open(path).unwrap();
|
||||
|
||||
if !self.is_empty() {
|
||||
panic!()
|
||||
}
|
||||
|
||||
self.cached_puts = btree;
|
||||
}
|
||||
|
||||
if self.cached_dels.is_empty() && self.cached_puts.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let cached_dels = mem::take(&mut self.cached_dels);
|
||||
self.db_multi_del(cached_dels)?;
|
||||
|
||||
let cached_puts = mem::take(&mut self.cached_puts);
|
||||
self.db_multi_put(cached_puts)?;
|
||||
|
||||
self.txn.set_root(ROOT_DB, self.db.db.into());
|
||||
|
||||
self.txn.commit()
|
||||
}
|
||||
|
||||
fn destroy(self) -> io::Result<()> {
|
||||
let path = self.path.to_owned();
|
||||
|
||||
drop(self);
|
||||
|
||||
fs::remove_file(&path)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
use std::{fs, io, path::Path};
|
||||
|
||||
use log::info;
|
||||
use snkrj::AnyDatabase;
|
||||
|
||||
use crate::structs::{Config, Date, Height};
|
||||
|
||||
use super::{AnyDatabase, Metadata};
|
||||
use super::Metadata;
|
||||
|
||||
pub trait AnyDatabaseGroup
|
||||
where
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::{
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::{
|
||||
parser::states::AddressCohortsDurableStates,
|
||||
@@ -15,7 +16,7 @@ use crate::{
|
||||
utils::time,
|
||||
};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = u32;
|
||||
type Value = AddressData;
|
||||
@@ -25,6 +26,7 @@ type Database = _Database<Key, Value>;
|
||||
pub struct AddressIndexToAddressData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
#[allocative(skip)]
|
||||
pub map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -45,10 +47,10 @@ impl DerefMut for AddressIndexToAddressData {
|
||||
pub const ADDRESS_INDEX_DB_MAX_SIZE: usize = 250_000;
|
||||
|
||||
impl AddressIndexToAddressData {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
@@ -59,16 +61,16 @@ impl AddressIndexToAddressData {
|
||||
|
||||
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
|
||||
/// Though it makes it easy to use with rayon.
|
||||
pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index).unwrap().get_from_puts(key)
|
||||
self.get(&db_index).unwrap().get_from_ram(key)
|
||||
}
|
||||
|
||||
pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index).unwrap().db_get(key)
|
||||
self.get(&db_index).unwrap().get_from_disk(key)
|
||||
}
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
@@ -99,7 +101,7 @@ impl AddressIndexToAddressData {
|
||||
let mut s = AddressCohortsDurableStates::default();
|
||||
|
||||
database
|
||||
.iter()
|
||||
.iter_disk()
|
||||
.map(|r| r.unwrap().1)
|
||||
.for_each(|address_data| s.increment(address_data).unwrap());
|
||||
|
||||
|
||||
@@ -7,12 +7,11 @@ use std::{
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Config, EmptyAddressData};
|
||||
|
||||
use super::{
|
||||
AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE,
|
||||
};
|
||||
use super::{AnyDatabaseGroup, Metadata, ADDRESS_INDEX_DB_MAX_SIZE};
|
||||
|
||||
type Key = u32;
|
||||
type Value = EmptyAddressData;
|
||||
@@ -22,6 +21,7 @@ type Database = _Database<Key, Value>;
|
||||
pub struct AddressIndexToEmptyAddressData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -40,10 +40,10 @@ impl DerefMut for AddressIndexToEmptyAddressData {
|
||||
}
|
||||
|
||||
impl AddressIndexToEmptyAddressData {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
@@ -54,13 +54,13 @@ impl AddressIndexToEmptyAddressData {
|
||||
|
||||
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
|
||||
/// Though it makes it easy to use with rayon.
|
||||
pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index).and_then(|db| db.get_from_puts(key))
|
||||
self.get(&db_index).and_then(|db| db.get_from_ram(key))
|
||||
}
|
||||
|
||||
pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index)
|
||||
@@ -68,7 +68,7 @@ impl AddressIndexToEmptyAddressData {
|
||||
dbg!(&self.map.keys(), &key, &db_index);
|
||||
panic!()
|
||||
})
|
||||
.db_get(key)
|
||||
.get_from_disk(key)
|
||||
}
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
|
||||
@@ -6,10 +6,11 @@ use std::{
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database};
|
||||
|
||||
use crate::structs::{Address, Config, U8x19, U8x31};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Value = u32;
|
||||
type U8x19Database = Database<U8x19, Value>;
|
||||
@@ -33,16 +34,27 @@ pub struct AddressToAddressIndex {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
#[allocative(skip)]
|
||||
p2pk: BTreeMap<u16, P2PKDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2pkh: BTreeMap<u16, P2PKHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2sh: BTreeMap<u16, P2SHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2wpkh: BTreeMap<u16, P2WPKHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2wsh: BTreeMap<u16, P2WSHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2tr: BTreeMap<u16, P2TRDatabase>,
|
||||
#[allocative(skip)]
|
||||
op_return: Option<OpReturnDatabase>,
|
||||
#[allocative(skip)]
|
||||
push_only: Option<PushOnlyDatabase>,
|
||||
#[allocative(skip)]
|
||||
unknown: Option<UnknownDatabase>,
|
||||
#[allocative(skip)]
|
||||
empty: Option<EmptyDatabase>,
|
||||
#[allocative(skip)]
|
||||
multisig: Option<MultisigDatabase>,
|
||||
}
|
||||
|
||||
@@ -103,19 +115,19 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unsafe_get_from_puts(&self, address: &Address) -> Option<&Value> {
|
||||
pub fn get_from_ram(&self, address: &Address) -> Option<&Value> {
|
||||
match address {
|
||||
Address::Empty(key) => self.empty.as_ref().unwrap().get_from_puts(key),
|
||||
Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_puts(key),
|
||||
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_puts(key),
|
||||
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_puts(key),
|
||||
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_puts(key),
|
||||
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::Empty(key) => self.empty.as_ref().unwrap().get_from_ram(key),
|
||||
Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_ram(key),
|
||||
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_ram(key),
|
||||
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_ram(key),
|
||||
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_ram(key),
|
||||
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_ram(key),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use allocative::Allocative;
|
||||
|
||||
mod _database;
|
||||
mod _trait;
|
||||
mod address_index_to_address_data;
|
||||
mod address_index_to_empty_address_data;
|
||||
@@ -10,7 +9,6 @@ mod txid_to_tx_data;
|
||||
mod txout_index_to_address_index;
|
||||
mod txout_index_to_amount;
|
||||
|
||||
pub use _database::*;
|
||||
use _trait::*;
|
||||
pub use address_index_to_address_data::*;
|
||||
pub use address_index_to_empty_address_data::*;
|
||||
@@ -19,6 +17,7 @@ use itertools::Itertools;
|
||||
use log::info;
|
||||
use metadata::*;
|
||||
use rayon::iter::{IntoParallelIterator, ParallelIterator};
|
||||
use snkrj::AnyDatabase;
|
||||
pub use txid_to_tx_data::*;
|
||||
pub use txout_index_to_address_index::*;
|
||||
pub use txout_index_to_amount::*;
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use biter::bitcoin::Txid;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Config, TxData, U8x31};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = U8x31;
|
||||
type Value = TxData;
|
||||
@@ -21,23 +21,10 @@ type Database = _Database<Key, Value>;
|
||||
pub struct TxidToTxData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<u16, Database>,
|
||||
}
|
||||
|
||||
impl Deref for TxidToTxData {
|
||||
type Target = BTreeMap<u16, Database>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.map
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for TxidToTxData {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.map
|
||||
}
|
||||
}
|
||||
|
||||
impl TxidToTxData {
|
||||
pub fn insert(&mut self, txid: &Txid, tx_index: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
@@ -47,53 +34,41 @@ impl TxidToTxData {
|
||||
self.open_db(txid).insert(txid_key, tx_index)
|
||||
}
|
||||
|
||||
// pub fn safe_get(&mut self, txid: &Txid) -> Option<&Value> {
|
||||
// let txid_key = Self::txid_to_key(txid);
|
||||
// self.open_db(txid).get(&txid_key)
|
||||
// }
|
||||
|
||||
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed.
|
||||
/// Though it makes it easy to use with rayon
|
||||
pub fn unsafe_get(&self, txid: &Txid) -> Option<&Value> {
|
||||
pub fn get(&self, txid: &Txid) -> Option<&Value> {
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
let db_index = Self::db_index(txid);
|
||||
|
||||
self.get(&db_index).unwrap().get(&txid_key)
|
||||
self.map.get(&db_index).unwrap().get(&txid_key)
|
||||
}
|
||||
|
||||
// pub fn unsafe_get_from_puts(&self, txid: &Txid) -> Option<&Value> {
|
||||
// let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
// let db_index = Self::db_index(txid);
|
||||
|
||||
// self.get(&db_index).unwrap().get_from_puts(&txid_key)
|
||||
// }
|
||||
|
||||
pub fn unsafe_get_mut_from_puts(&mut self, txid: &Txid) -> Option<&mut Value> {
|
||||
pub fn get_mut_from_ram(&mut self, txid: &Txid) -> Option<&mut Value> {
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
let db_index = Self::db_index(txid);
|
||||
|
||||
self.get_mut(&db_index)
|
||||
self.map
|
||||
.get_mut(&db_index)
|
||||
.unwrap()
|
||||
.get_mut_from_puts(&txid_key)
|
||||
.get_mut_from_ram(&txid_key)
|
||||
}
|
||||
|
||||
pub fn remove_from_db(&mut self, txid: &Txid) {
|
||||
pub fn remove_later_from_disk(&mut self, txid: &Txid) {
|
||||
self.metadata.called_remove();
|
||||
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
self.open_db(txid).db_remove(&txid_key);
|
||||
self.open_db(txid).remove_later_from_disk(&txid_key);
|
||||
}
|
||||
|
||||
pub fn remove_from_puts(&mut self, txid: &Txid) {
|
||||
pub fn remove_from_ram(&mut self, txid: &Txid) {
|
||||
self.metadata.called_remove();
|
||||
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
self.open_db(txid).remove_from_puts(&txid_key);
|
||||
self.open_db(txid).remove_from_ram(&txid_key);
|
||||
}
|
||||
|
||||
pub fn update(&mut self, txid: &Txid, tx_data: TxData) {
|
||||
@@ -111,7 +86,7 @@ impl TxidToTxData {
|
||||
#[inline(always)]
|
||||
pub fn _open_db(&mut self, db_index: u16) -> &mut Database {
|
||||
let path = self.path.to_owned();
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
self.map.entry(db_index).or_insert_with(|| {
|
||||
let path = path.join(db_index.to_string());
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
|
||||
@@ -7,10 +7,11 @@ use std::{
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Config, TxoutIndex};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = TxoutIndex;
|
||||
type Value = u32;
|
||||
@@ -20,6 +21,7 @@ type Database = _Database<Key, Value>;
|
||||
pub struct TxoutIndexToAddressIndex {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -40,20 +42,12 @@ impl DerefMut for TxoutIndexToAddressIndex {
|
||||
const DB_MAX_SIZE: usize = 10_000_000_000;
|
||||
|
||||
impl TxoutIndexToAddressIndex {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
|
||||
// self.open_db(key).remove_from_puts(key).map(|v| {
|
||||
// self.metadata.called_remove();
|
||||
|
||||
// v
|
||||
// })
|
||||
// }
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.metadata.called_remove();
|
||||
|
||||
|
||||
@@ -7,10 +7,11 @@ use std::{
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Amount, Config, TxoutIndex};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = TxoutIndex;
|
||||
type Value = Amount;
|
||||
@@ -20,6 +21,7 @@ type Database = _Database<Key, Value>;
|
||||
pub struct TxoutIndexToAmount {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -40,20 +42,12 @@ impl DerefMut for TxoutIndexToAmount {
|
||||
const DB_MAX_SIZE: usize = 10_000_000_000;
|
||||
|
||||
impl TxoutIndexToAmount {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
|
||||
// self.open_db(key).remove_from_puts(key).map(|v| {
|
||||
// self.metadata.called_remove();
|
||||
|
||||
// v
|
||||
// })
|
||||
// }
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.metadata.called_remove();
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use allocative::Allocative;
|
||||
use color_eyre::eyre::eyre;
|
||||
use sanakirja::{direct_repr, Storable, UnsizedStorable};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
|
||||
use super::{AddressType, Amount, EmptyAddressData, LiquidityClassification, Price};
|
||||
|
||||
|
||||
@@ -12,8 +12,8 @@ use bincode::{
|
||||
};
|
||||
use biter::bitcoin::Amount as BitcoinAmount;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use sanakirja::{direct_repr, Storable, UnsizedStorable};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
|
||||
use super::Height;
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::fmt::Debug;
|
||||
|
||||
use allocative::Allocative;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use sanakirja::{direct_repr, Storable, UnsizedStorable};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
|
||||
#[derive(
|
||||
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative,
|
||||
|
||||
@@ -74,6 +74,10 @@ impl Date {
|
||||
self.month() == 1
|
||||
}
|
||||
|
||||
pub fn is_june(&self) -> bool {
|
||||
self.month() == 6
|
||||
}
|
||||
|
||||
pub fn is_first_of_month(&self) -> bool {
|
||||
self.day() == 1
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use allocative::Allocative;
|
||||
use sanakirja::{direct_repr, Storable, UnsizedStorable};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
|
||||
use super::{AddressData, AddressType, Amount};
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use allocative::Allocative;
|
||||
use sanakirja::{direct_repr, Storable, UnsizedStorable};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
|
||||
use super::BlockPath;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use allocative::Allocative;
|
||||
use bincode::{Decode, Encode};
|
||||
use sanakirja::{direct_repr, Storable, UnsizedStorable};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Encode, Decode, Allocative)]
|
||||
pub struct TxoutIndex {
|
||||
|
||||
Reference in New Issue
Block a user