snkrj: move database struct to its own crate

This commit is contained in:
nym21
2024-12-14 14:55:44 +01:00
parent c883ed19d6
commit 50c77b51db
28 changed files with 702 additions and 349 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2024 Biter
Copyright (c) 2024 biter
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
+242
View File
@@ -0,0 +1,242 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "autocfg"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]]
name = "libc"
version = "0.2.168"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"
[[package]]
name = "lock_api"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "memmap2"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
dependencies = [
"libc",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]]
name = "proc-macro2"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
]
[[package]]
name = "sanakirja"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81aaf70d064e2122209f04d01fd91e8908e7a327b516236e1cbc0c3f34ac6d11"
dependencies = [
"fs2",
"log",
"memmap2",
"parking_lot",
"sanakirja-core",
"serde",
"thiserror",
]
[[package]]
name = "sanakirja-core"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "serde"
version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "snkrj"
version = "0.1.0"
dependencies = [
"sanakirja",
]
[[package]]
name = "syn"
version = "2.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "unicode-ident"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+12
View File
@@ -0,0 +1,12 @@
[package]
name = "snkrj"
description = "A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap"
version = "0.1.0"
license = "MIT"
repository = "https://github.com/kibo-money/kibo/tree/main/crates/snkrj"
keywords = ["database", "sanakirja", "btreemap"]
categories = ["database"]
edition = "2021"
[dependencies]
sanakirja = "1.4.3"
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 snkrj
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+37
View File
@@ -0,0 +1,37 @@
# snkrj
A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap.
## Example
```rust
use snkrj::{AnyDatabase, Database};
fn main() {
let path = std::env::temp_dir().join("./db");
let database: Database<i32, i32> = Database::open(path.clone()).unwrap();
let _ = database.destroy();
let mut database: Database<i32, i32> = Database::open(path.clone()).unwrap();
database.insert(64, 128);
database.export(false).unwrap();
let mut database: Database<i32, i32> = Database::open(path).unwrap();
database.insert(1, 2);
database.insert(128, 256);
println!("iter_ram:");
database.iter_ram().for_each(|pair| {
println!("{:?}", pair);
});
println!("iter_disk:");
database.iter_disk().for_each(|pair| {
println!("{:?}", pair.unwrap());
});
println!("iter_ram_then_disk:");
database.iter_ram_then_disk().for_each(|pair| {
println!("{:?}", pair);
});
database.export(false).unwrap();
}
```
+252
View File
@@ -0,0 +1,252 @@
// https://docs.rs/sanakirja/latest/sanakirja/index.html
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
use std::{
collections::{BTreeMap, BTreeSet},
fmt::Debug,
fs, io, mem,
path::PathBuf,
result::Result,
};
use sanakirja::btree::{page, Db_};
pub use sanakirja::*;
///
/// A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap.
///
/// The state of the tree is uncommited until `.export()` is called during which it is unsafe to stop the program.
///
pub struct Database<Key, Value>
where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq,
{
path: PathBuf,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
db: Db_<Key, Value, page::Page<Key, Value>>,
txn: MutTxn<Env, ()>,
}
const ROOT_DB: usize = 0;
const PAGE_SIZE: u64 = 4096;
impl<Key, Value> Database<Key, Value>
where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq,
{
/// Open a database without a lock file where only one instance is safe to open.
pub fn open(path: PathBuf) -> Result<Self, Error> {
let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? };
let mut txn = Env::mut_txn_begin(env)?;
let db = txn
.root_db(ROOT_DB)
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
Ok(Self {
path,
puts: BTreeMap::default(),
dels: BTreeSet::default(),
db,
txn,
})
}
#[inline]
pub fn get(&self, key: &Key) -> Option<&Value> {
if let Some(cached_put) = self.get_from_ram(key) {
return Some(cached_put);
}
self.get_from_disk(key)
}
/// Get only from the uncommited tree (ram) without checking the database (disk)
#[inline]
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
self.puts.get(key)
}
/// Get mut only from the uncommited tree (ram) without checking the database (disk)
#[inline]
pub fn get_mut_from_ram(&mut self, key: &Key) -> Option<&mut Value> {
self.puts.get_mut(key)
}
/// Get only from the database (disk) without checking the uncommited tree (ram)
#[inline]
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
if let Some((key_found, v)) = option {
if key == key_found {
return Some(v);
}
}
None
}
#[inline]
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.dels.remove(&key);
self.insert_to_ram(key, value)
}
/// Insert without removing the key to the dels tree, so be sure that it hasn't added to the delete set
#[inline]
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
self.puts.insert(key, value)
}
#[inline]
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
self.dels.insert(key.clone());
self.puts.insert(key, value)
}
#[inline]
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.remove_from_ram(key).or_else(|| {
self.remove_later_from_disk(key);
None
})
}
/// Get only from the uncommited tree (ram) without checking the database (disk)
#[inline]
pub fn remove_from_ram(&mut self, key: &Key) -> Option<Value> {
self.puts.remove(key)
}
/// Add the key only to the dels tree without checking if it's present in the puts tree, only use if you are positive that you neither added nor updated an entry with this key
#[inline]
pub fn remove_later_from_disk(&mut self, key: &Key) {
self.dels.insert(key.clone());
}
#[inline]
pub fn is_empty(&self) -> bool {
self.iter_disk().next().is_none()
}
/// Iterate over key/value pairs from the uncommited tree (ram)
#[inline]
pub fn iter_ram(&self) -> std::collections::btree_map::Iter<'_, Key, Value> {
self.puts.iter()
}
/// Iterate over key/value pairs from the database (disk)
#[inline]
pub fn iter_disk(
&self,
) -> btree::Iter<'_, MutTxn<Env, ()>, Key, Value, page::Page<Key, Value>> {
btree::iter(&self.txn, &self.db, None).unwrap()
}
/// Iterate over key/value pairs
#[inline]
pub fn iter_ram_then_disk(&self) -> impl Iterator<Item = (&Key, &Value)> {
self.iter_ram().chain(self.iter_disk().map(|r| r.unwrap()))
}
/// Collect a **clone** of all uncommited key/value pairs (ram)
pub fn collect_ram(&self) -> BTreeMap<Key, Value>
where
Value: Clone,
{
self.puts.clone()
}
/// Collect a **clone** of all key/value pairs from the database (disk)
pub fn collect_disk(&self) -> BTreeMap<Key, Value>
where
Value: Clone,
{
self.iter_disk()
.map(|r| r.unwrap())
.map(|(key, value)| (key.clone(), value.clone()))
.collect::<_>()
}
}
pub trait AnyDatabase {
#[allow(unused)]
fn export(self, defragment: bool) -> Result<(), Error>;
fn boxed_export(self: Box<Self>, defragment: bool) -> Result<(), Error>;
#[allow(unused)]
fn destroy(self) -> io::Result<()>;
}
impl<Key, Value> AnyDatabase for Database<Key, Value>
where
Key: Ord + Clone + Debug + Storable,
Value: Storable + PartialEq + Clone,
{
/// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space
///
/// /!\ Do not kill the program while this function is runnning /!\
fn export(self, defragment: bool) -> Result<(), Error> {
Box::new(self).boxed_export(defragment)
}
/// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space
///
/// /!\ Do not kill the program while this function is runnning /!\
fn boxed_export(mut self: Box<Self>, defragment: bool) -> Result<(), Error> {
if defragment {
let mut btree = self.as_ref().collect_disk();
let path = self.path.to_owned();
self.dels.iter().for_each(|key| {
btree.remove(key);
});
btree.append(&mut self.puts);
self.destroy()?;
*self = Self::open(path).unwrap();
if !self.is_empty() {
panic!()
}
self.puts = btree;
}
if self.dels.is_empty() && self.puts.is_empty() {
return Ok(());
}
mem::take(&mut self.dels)
.into_iter()
.try_for_each(|key| -> Result<(), Error> {
btree::del(&mut self.txn, &mut self.db, &key, None)?;
Ok(())
})?;
mem::take(&mut self.puts).into_iter().try_for_each(
|(key, value)| -> Result<(), Error> {
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
},
)?;
self.txn.set_root(ROOT_DB, self.db.db.into());
self.txn.commit()
}
fn destroy(self) -> io::Result<()> {
let path = self.path.to_owned();
drop(self);
fs::remove_file(&path)
}
}
+29
View File
@@ -0,0 +1,29 @@
use snkrj::{AnyDatabase, Database};
fn main() {
let path = std::env::temp_dir().join("./db");
let database: Database<i32, i32> = Database::open(path.clone()).unwrap();
let _ = database.destroy();
let mut database: Database<i32, i32> = Database::open(path.clone()).unwrap();
database.insert(64, 128);
database.export(false).unwrap();
let mut database: Database<i32, i32> = Database::open(path).unwrap();
database.insert(1, 2);
database.insert(128, 256);
println!("iter_ram:");
database.iter_ram().for_each(|pair| {
println!("{:?}", pair);
});
println!("iter_disk:");
database.iter_disk().for_each(|pair| {
println!("{:?}", pair.unwrap());
});
println!("iter_ram_then_disk:");
database.iter_ram_then_disk().for_each(|pair| {
println!("{:?}", pair);
});
database.export(false).unwrap();
}