Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-08-04 22:53:05 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: fjall+lsm-tree+vecdb

Browse Source
This commit is contained in:
nym21
2026-08-02 22:47:43 +02:00
parent 9927ac782e
commit d61bdb1962
349 changed files with 2922 additions and 18868 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -27,6 +27,7 @@ _*
/*.txt
/*.csv
/tmp
**/.test
# Logs
*.log*
Cargo.lock
Generated
+18 -61
View File
@@ -861,9 +861,11 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "byteview"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d74937895e761d5984206f82ca8ff7725d0fd8021921011921894b41ab1b9f7"
version = "0.3.6"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "cast"
@@ -1462,18 +1464,6 @@ version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
[[package]]
name = "enum_dispatch"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "env_filter"
version = "2.0.0"
@@ -1567,7 +1557,6 @@ dependencies = [
"lsm-tree",
"lz4_flex",
"nanoid",
"rand 0.10.2",
"tempfile",
"test-log",
"xxhash-rust",
@@ -1789,12 +1778,6 @@ version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -2288,9 +2271,9 @@ dependencies = [
[[package]]
name = "libredox"
version = "0.1.18"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652"
checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa"
dependencies = [
"libc",
]
@@ -2333,18 +2316,16 @@ name = "lsm-tree"
version = "0.3.6"
dependencies = [
"byteorder-lite",
"bytes",
"byteview",
"criterion",
"crossbeam-skiplist",
"enum_dispatch",
"fs_extra",
"interval-heap",
"log",
"lz4_flex",
"nanoid",
"quick_cache",
"rand 0.9.5",
"rand 0.10.2",
"rustc-hash",
"self_cell",
"sfa",
@@ -2468,11 +2449,11 @@ dependencies = [
[[package]]
name = "nanoid"
version = "0.4.0"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8"
checksum = "8628de41fe064cc3f0cf07f3d299ee3e73521adaff72278731d5c8cae3797873"
dependencies = [
"rand 0.8.7",
"rand 0.9.5",
]
[[package]]
@@ -2624,9 +2605,9 @@ dependencies = [
[[package]]
name = "pco"
version = "1.0.2"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "553ccdc7f6999785559af4998c79712a5ab820e26b68bad9146609c19587ec82"
checksum = "386342cad4c6e97f081568e5d910ea7d871314c843aa8fc564f2a6b64cab9456"
dependencies = [
"better_io",
"dtype_dispatch",
@@ -2781,12 +2762,12 @@ dependencies = [
[[package]]
name = "quick_cache"
version = "0.6.24"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
"hashbrown 0.17.1",
]
[[package]]
@@ -2826,24 +2807,13 @@ dependencies = [
"ptr_meta",
]
[[package]]
name = "rand"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.4",
]
[[package]]
name = "rand"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
dependencies = [
"rand_chacha 0.9.0",
"rand_chacha",
"rand_core 0.9.5",
]
@@ -2858,16 +2828,6 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.4",
]
[[package]]
name = "rand_chacha"
version = "0.9.0"
@@ -2883,9 +2843,6 @@ name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom 0.2.17",
]
[[package]]
name = "rand_core"
@@ -4047,7 +4004,7 @@ name = "vecdb_derive"
version = "0.3.6"
dependencies = [
"quote",
"syn 2.0.119",
"syn 3.0.3",
"tempfile",
"vecdb",
]
Cargo.toml
+3 -3
View File
@@ -59,7 +59,7 @@ brk_traversable = { version = "0.3.6", path = "crates/brk_traversable", features
brk_traversable_derive = { version = "0.3.6", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.6", path = "crates/brk_types" }
brk_website = { version = "0.3.6", path = "crates/brk_website" }
byteview = "0.10.2"
byteview = { version = "0.3.6", path = "crates/byteview" }
color-eyre = "0.6.5"
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
corepc-types = { version = "0.15.0", features = ["std"], default-features = false }
@@ -70,10 +70,10 @@ jiff = { version = "0.2.35", features = ["perf-inline", "tz-system"], default-fe
libc = "0.2"
log = "0.4.33"
lsm-tree = { path = "crates/lsm-tree", version = "0.3.6", default-features = false }
lz4_flex = { version = "0.13.1", default-features = false }
lz4_flex = { version = "=0.13.1", default-features = false }
owo-colors = "4.3.0"
parking_lot = "0.12.5"
pco = "1.0.2"
pco = "1.0.3"
quickmatch = { path = "crates/quickmatch", version = "0.3.6" }
rawdb = { path = "crates/rawdb", version = "0.3.6" }
rayon = "1.12.0"
crates/brk_store/src/lib.rs
+7 -10
View File
@@ -299,14 +299,14 @@ where
let mut ingestion = keyspace.start_ingestion()?;
// FxHashMap/FxHashSet keep keys unique and disjoint; sorting therefore
// proves the strict ordering required by the prevalidated writer.
// proves the strict ordering required by ingestion.
for item in items {
match item {
Item::Value { key, value } => {
ingestion.write_prevalidated(ByteView::from(key), ByteView::from(value))?;
ingestion.write(ByteView::from(key), ByteView::from(value))?;
}
Item::Tomb(key) => {
ingestion.write_prevalidated_weak_tombstone(ByteView::from(key))?;
ingestion.write_weak_tombstone(ByteView::from(key))?;
}
}
}
@@ -335,22 +335,19 @@ where
(Some((put_key, _)), Some(del_key)) => match put_key.cmp(del_key) {
Ordering::Less => {
let (key, value) = puts.next().unwrap();
ingestion.write_prevalidated(ByteView::from(key), ByteView::from(value))?;
ingestion.write(ByteView::from(key), ByteView::from(value))?;
}
Ordering::Greater => {
ingestion.write_prevalidated_weak_tombstone(ByteView::from(
dels.next().unwrap(),
))?;
ingestion.write_weak_tombstone(ByteView::from(dels.next().unwrap()))?;
}
Ordering::Equal => unreachable!("key is both inserted and deleted"),
},
(Some(_), None) => {
let (key, value) = puts.next().unwrap();
ingestion.write_prevalidated(ByteView::from(key), ByteView::from(value))?;
ingestion.write(ByteView::from(key), ByteView::from(value))?;
}
(None, Some(_)) => {
ingestion
.write_prevalidated_weak_tombstone(ByteView::from(dels.next().unwrap()))?;
ingestion.write_weak_tombstone(ByteView::from(dels.next().unwrap()))?;
}
(None, None) => break,
}
crates/byteview/Cargo.toml
+21
View File
@@ -0,0 +1,21 @@
[package]
name = "byteview"
description = "Thin, immutable zero-copy slice type"
license = "MIT OR Apache-2.0"
version.workspace = true
edition.workspace = true
readme = "README.md"
repository.workspace = true
homepage.workspace = true
categories = ["data-structures"]
keywords = ["german-string", "string-view", "byte-slice"]
[features]
default = []
serde = ["dep:serde"]
[dependencies]
serde = { workspace = true, optional = true }
[dev-dependencies]
serde_json = { workspace = true }
crates/byteview/LICENSE-APACHE
+176
View File
@@ -0,0 +1,176 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
crates/byteview/LICENSE-MIT
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 fjall-rs
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
crates/byteview/README.md
+76
View File
@@ -0,0 +1,76 @@
# byteview
[![CI](https://github.com/fjall-rs/byteview/actions/workflows/test.yml/badge.svg)](https://github.com/fjall-rs/byteview/actions/workflows/test.yml)
[![CI](https://github.com/fjall-rs/byteview/actions/workflows/miri.yml/badge.svg)](https://github.com/fjall-rs/byteview/actions/workflows/miri.yml)
[![docs.rs](https://img.shields.io/docsrs/byteview?color=green)](https://docs.rs/byteview)
[![Crates.io](https://img.shields.io/crates/v/byteview?color=blue)](https://crates.io/crates/byteview)
![MSRV](https://img.shields.io/badge/MSRV-1.87-blue)
An immutable byte slice that may be inlined, and can be partially cloned without heap allocation.
Think of it as a specialized `Arc<[u8]>` that can be inlined (skip allocation for small values) and no weak count.
![Memory layout](./byteview.png)
`byteview` was originally designed to speed up deserialization in [`lsm-tree`](https://github.com/fjall-rs/lsm-tree), allow inlining of small values and reduce memory usage compared to Arc'd slices.
Values with a known length can be constructed 2-2.5x faster than using `Arc<[u8]>`:
![Constructor benchmark](ctor_bench.png)
## Sponsors
<a href="https://sqlsync.dev">
<picture>
<source width="240" alt="Orbitinghail" media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/fjall-rs/fjall-rs.github.io/d22fcb1e6966ce08327ea3bf6cf2ea86a840b071/public/logos/orbitinghail.svg" />
<source width="240" alt="Orbitinghail" media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/fjall-rs/fjall-rs.github.io/d22fcb1e6966ce08327ea3bf6cf2ea86a840b071/public/logos/orbitinghail_dark.svg" />
<img width="240" alt="Orbitinghail" src="https://raw.githubusercontent.com/fjall-rs/fjall-rs.github.io/d22fcb1e6966ce08327ea3bf6cf2ea86a840b071/public/logos/orbitinghail_dark.svg" />
</picture>
</a>
## Memory usage
Allocating 200M "" (len=0) strings:
| Struct | Memory Usage |
|-----------------|--------------|
| `Arc<[u8]>` | 9.6 GB |
| `tokio::Bytes` | 6.4 GB |
| `ByteView` | 4.8 GB |
Allocating 200M "helloworld" (len=10) strings:
| Struct | Memory Usage |
|-----------------|--------------|
| `Arc<[u8]>` | 12.8 GB |
| `tokio::Bytes` | 12.8 GB |
| `ByteView` | 4.8 GB |
Allocating 100M "helloworldhelloworld" (len=20) strings:
| Struct | Memory Usage |
|-----------------|--------------|
| `Arc<[u8]>` | 6.4 GB |
| `tokio::Bytes` | 6.4 GB |
| `ByteView` | 2.4 GB |
Allocating 50M "helloworldhelloworldhelloworldhelloworld" (len=30) strings:
| Struct | Memory Usage |
|-----------------|--------------|
| `Arc<[u8]>` | 4.0 GB |
| `tokio::Bytes` | 4.0 GB |
| `ByteView` | 3.6 GB |
Allocating 500k `"helloworld".repeat(1000)` (len=10'000) strings:
| Struct | Memory Usage |
|-----------------|--------------|
| `Arc<[u8]>` | 5 GB |
| `tokio::Bytes` | 5 GB |
| `ByteView` | 5 GB |
## Run fuzz tests
```bash
cargo +nightly fuzz run fuzz_target_1
```
crates/byteview/src/builder.rs
+33
View File
@@ -0,0 +1,33 @@
use crate::ByteView;
/// A builder for a [`ByteView`] that allows mutation before freezing it.
pub struct Builder(ByteView);
impl Builder {
/// Creates a new builder.
#[must_use]
pub const fn new(inner: ByteView) -> Self {
Self(inner)
}
/// Converts the builder into a [`ByteView`], making it immutable.
#[must_use]
pub fn freeze(mut self) -> ByteView {
self.0.update_prefix();
self.0
}
}
impl std::ops::Deref for Builder {
type Target = [u8];
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for Builder {
fn deref_mut(&mut self) -> &mut Self::Target {
self.0.get_mut_slice()
}
}
crates/byteview/src/byteview.rs
+1283
View File
File diff suppressed because it is too large Load Diff
crates/byteview/src/lib.rs
+57
View File
@@ -0,0 +1,57 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
//! An immutable byte slice that may be inlined, and can be partially cloned without heap allocation.
//!
//! The length is limited to 2^32 bytes (4 GiB).
//!
//! ```
//! # use byteview::ByteView;
//! let slice = ByteView::from("helloworld_thisisaverylongstring");
//!
//! // No heap allocation - increases the ref count like an Arc<[u8]>
//! let full_copy = slice.clone();
//! drop(full_copy);
//!
//! // No heap allocation - increases the ref count like an Arc<[u8]>, but we only get a subslice
//! let copy = slice.slice(11..);
//! assert_eq!(b"thisisaverylongstring", &*copy);
//!
//! // No heap allocation - if the slice is small enough, it will be inlined into the struct...
//! let copycopy = copy.slice(0..4);
//! assert_eq!(b"this", &*copycopy);
//!
//! // ...so no ref count incrementing is done
//! assert_eq!(2, slice.ref_count());
//!
//! drop(copy);
//! assert_eq!(1, slice.ref_count());
//!
//! drop(copycopy);
//! assert_eq!(1, slice.ref_count());
//!
//! // Our original slice will be automatically freed if all slices vanish
//! drop(slice);
//! ```
#![deny(clippy::all, missing_docs, clippy::cargo)]
#![deny(clippy::unwrap_used)]
#![deny(clippy::indexing_slicing)]
#![warn(
clippy::pedantic,
clippy::nursery,
clippy::expect_used,
clippy::unwrap_used,
clippy::indexing_slicing,
clippy::needless_lifetimes
)]
mod builder;
mod byteview;
mod strview;
pub use {byteview::ByteView, strview::StrView};
#[doc(hidden)]
pub use byteview::{Builder, Mutator};
crates/byteview/src/strview.rs
+429
View File
@@ -0,0 +1,429 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::ByteView;
use std::{ops::Deref, sync::Arc};
/// An immutable, UTF-8–encoded string slice
///
/// Will be inlined (no pointer dereference or heap allocation)
/// if it is 20 characters or shorter (on a 64-bit system).
///
/// A single heap allocation will be shared between multiple strings.
/// Even substrings of that heap allocation can be cloned without additional heap allocation.
///
/// Uses [`ByteView`] internally, but derefs as [`&str`].
#[repr(C)]
#[derive(Default, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct StrView(ByteView);
impl std::fmt::Display for StrView {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &**self)
}
}
impl std::fmt::Debug for StrView {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", &**self)
}
}
impl Deref for StrView {
type Target = str;
fn deref(&self) -> &Self::Target {
// SAFETY: Constructor takes a &str
unsafe { std::str::from_utf8_unchecked(&self.0) }
}
}
impl std::hash::Hash for StrView {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.deref().hash(state);
}
}
impl StrView {
/// Creates a new string from an existing byte string.
///
/// Will heap-allocate the string if it has at least length 13.
///
/// # Panics
///
/// Panics if the length does not fit in a u32 (4 GiB).
#[must_use]
pub fn new(s: &str) -> Self {
Self(ByteView::new(s.as_bytes()))
}
#[doc(hidden)]
#[must_use]
#[allow(clippy::missing_const_for_fn)]
pub unsafe fn from_raw(view: ByteView) -> Self {
Self(view)
}
/// Clones the contents of this string into an independently tracked string.
#[must_use]
pub fn to_detached(&self) -> Self {
Self::new(self)
}
/// Clones the given range of the existing string without heap allocation.
#[must_use]
pub fn slice(&self, range: impl std::ops::RangeBounds<usize>) -> Self {
Self(self.0.slice(range))
}
/// Returns `true` if the string is empty.
#[must_use]
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Returns the amount of bytes in the string.
#[must_use]
pub fn len(&self) -> usize {
self.0.len()
}
/// Returns `true` if `needle` is a prefix of the string or equal to the string.
#[must_use]
pub fn starts_with(&self, needle: &str) -> bool {
self.0.starts_with(needle.as_bytes())
}
}
impl std::borrow::Borrow<str> for StrView {
fn borrow(&self) -> &str {
self
}
}
impl AsRef<str> for StrView {
fn as_ref(&self) -> &str {
self
}
}
impl From<&str> for StrView {
fn from(value: &str) -> Self {
Self::new(value)
}
}
impl From<String> for StrView {
fn from(value: String) -> Self {
Self::new(&value)
}
}
impl From<Arc<str>> for StrView {
fn from(value: Arc<str>) -> Self {
Self::new(&value)
}
}
impl TryFrom<ByteView> for StrView {
type Error = std::str::Utf8Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
std::str::from_utf8(&value)?;
Ok(Self(value))
}
}
impl From<StrView> for ByteView {
fn from(val: StrView) -> Self {
val.0
}
}
#[cfg(feature = "serde")]
mod serde {
use super::StrView;
use serde::de::{self, Visitor};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use std::ops::Deref;
impl Serialize for StrView {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(self.deref())
}
}
impl<'de> Deserialize<'de> for StrView {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct StrViewVisitor;
impl<'de> Visitor<'de> for StrViewVisitor {
type Value = StrView;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a string")
}
fn visit_str<E>(self, v: &str) -> Result<StrView, E>
where
E: de::Error,
{
Ok(StrView::new(v))
}
}
deserializer.deserialize_str(StrViewVisitor)
}
}
}
#[cfg(test)]
mod tests {
use super::StrView;
use std::collections::HashMap;
#[cfg(feature = "serde")]
#[test]
fn serde_roundtrip() {
let a = StrView::from("abcdef");
let b: StrView = serde_json::from_slice(&serde_json::to_vec(&a).unwrap()).unwrap();
assert_eq!(a, b);
}
#[test]
fn strview_hash() {
let a = StrView::from("abcdef");
let mut map = HashMap::new();
map.insert(a, 0);
assert!(map.contains_key("abcdef"));
}
#[test]
fn cmp_misc_1() {
let a = StrView::from("abcdef");
let b = StrView::from("abcdefhelloworldhelloworld");
assert!(a < b);
}
#[test]
fn nostr() {
let slice = StrView::from("");
assert_eq!(0, slice.len());
assert_eq!(&*slice, "");
}
#[test]
fn default_str() {
let slice = StrView::default();
assert_eq!(0, slice.len());
assert_eq!(&*slice, "");
}
#[test]
fn short_str() {
let slice = StrView::from("abcdef");
assert_eq!(6, slice.len());
assert_eq!(&*slice, "abcdef");
}
#[test]
#[cfg(target_pointer_width = "64")]
fn medium_str() {
let slice = StrView::from("abcdefabcdef");
assert_eq!(12, slice.len());
assert_eq!(&*slice, "abcdefabcdef");
}
#[test]
#[cfg(target_pointer_width = "64")]
fn medium_long_str() {
let slice = StrView::from("abcdefabcdefabcdabcd");
assert_eq!(20, slice.len());
assert_eq!(&*slice, "abcdefabcdefabcdabcd");
}
#[test]
#[cfg(target_pointer_width = "64")]
fn medium_str_clone() {
let slice = StrView::from("abcdefabcdefabcdefa");
#[allow(clippy::redundant_clone)]
let copy = slice.clone();
assert_eq!(slice, copy);
}
#[test]
fn long_str() {
let slice = StrView::from("abcdefabcdefabcdefababcd");
assert_eq!(24, slice.len());
assert_eq!(&*slice, "abcdefabcdefabcdefababcd");
}
#[test]
fn long_str_clone() {
let slice = StrView::from("abcdefabcdefabcdefababcd");
#[allow(clippy::redundant_clone)]
let copy = slice.clone();
assert_eq!(slice, copy);
}
#[test]
fn long_str_slice_full() {
let slice = StrView::from("helloworld_thisisalongstring");
let copy = slice.slice(..);
assert_eq!(copy, slice);
}
#[test]
#[cfg(target_pointer_width = "64")]
fn long_str_slice() {
let slice = StrView::from("helloworld_thisisalongstring");
let copy = slice.slice(11..);
assert_eq!("thisisalongstring", &*copy);
}
#[test]
#[cfg(target_pointer_width = "64")]
fn long_str_slice_twice() {
let slice = StrView::from("helloworld_thisisalongstring");
let copy = slice.slice(11..);
assert_eq!("thisisalongstring", &*copy);
let copycopy = copy.slice(..);
assert_eq!(copy, copycopy);
}
#[test]
#[cfg(target_pointer_width = "64")]
fn long_str_slice_downgrade() {
let slice = StrView::from("helloworld_thisisalongstring");
let copy = slice.slice(11..);
assert_eq!("thisisalongstring", &*copy);
let copycopy = copy.slice(0..4);
assert_eq!("this", &*copycopy);
{
let copycopy = copy.slice(0..=4);
assert_eq!("thisi", &*copycopy);
assert_eq!('t', copycopy.chars().next().unwrap());
}
}
#[test]
fn short_str_clone() {
let slice = StrView::from("abcdef");
let copy = slice.clone();
assert_eq!(slice, copy);
drop(slice);
assert_eq!(&*copy, "abcdef");
}
#[test]
fn short_str_slice_full() {
let slice = StrView::from("abcdef");
let copy = slice.slice(..);
assert_eq!(slice, copy);
drop(slice);
assert_eq!(&*copy, "abcdef");
}
#[test]
fn short_str_slice_part() {
let slice = StrView::from("abcdef");
let copy = slice.slice(3..);
drop(slice);
assert_eq!(&*copy, "def");
}
#[test]
fn short_str_slice_empty() {
let slice = StrView::from("abcdef");
let copy = slice.slice(0..0);
drop(slice);
assert_eq!(&*copy, "");
}
#[test]
fn tiny_str_starts_with() {
let a = StrView::from("abc");
assert!(a.starts_with("ab"));
assert!(!a.starts_with("b"));
}
#[test]
fn long_str_starts_with() {
let a = StrView::from("abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdef");
assert!(a.starts_with("abcdef"));
assert!(!a.starts_with("def"));
}
#[test]
fn tiny_str_cmp() {
let a = StrView::from("abc");
let b = StrView::from("def");
assert!(a < b);
}
#[test]
fn tiny_str_eq() {
let a = StrView::from("abc");
let b = StrView::from("def");
assert!(a != b);
}
#[test]
fn long_str_eq() {
let a = StrView::from("abcdefabcdefabcdefabcdef");
let b = StrView::from("xycdefabcdefabcdefabcdef");
assert!(a != b);
}
#[test]
fn long_str_cmp() {
let a = StrView::from("abcdefabcdefabcdefabcdef");
let b = StrView::from("xycdefabcdefabcdefabcdef");
assert!(a < b);
}
#[test]
fn long_str_eq_2() {
let a = StrView::from("abcdefabcdefabcdefabcdef");
let b = StrView::from("abcdefabcdefabcdefabcdef");
assert!(a == b);
}
#[test]
fn long_str_cmp_2() {
let a = StrView::from("abcdefabcdefabcdefabcdef");
let b = StrView::from("abcdefabcdefabcdefabcdeg");
assert!(a < b);
}
#[test]
fn long_str_cmp_3() {
let a = StrView::from("abcdefabcdefabcdefabcde");
let b = StrView::from("abcdefabcdefabcdefabcdef");
assert!(a < b);
}
}
crates/fjall/.config/nextest.toml
-2
View File
@@ -1,2 +0,0 @@
[profile.default]
slow-timeout = { period = "5s", terminate-after = 3 }
crates/fjall/.gitignore
-24
View File
@@ -1,24 +0,0 @@
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
mutants
mutants.out
.fjall_data
.data
.test
/old_*
.directory
crates/fjall/.rustfmt.toml
-3
View File
@@ -1,3 +0,0 @@
reorder_imports = true
# group_imports = "StdExternalCrate"
# imports_granularity = "crate"
crates/fjall/.vscode/settings.json
Vendored
-3
View File
@@ -1,3 +0,0 @@
{
"rust-analyzer.showUnlinkedFileNotification": false
}
crates/fjall/CHANGELOG.md
-50
View File
@@ -1,50 +0,0 @@
# 3.1.0
- [feat] Implemented support for compaction filters (custom logic during compactions)
- [msrv] Reduced MSRV to 1.90
# 3.0.0
- [feat] Implemented new block format in `lsm-tree`
- [feat] Bookkeep LSM-tree changes (flushes, compactions) in `Version` history
- [feat] Prefix truncation inside data & index blocks
- [feat] Allow unpinning filter blocks
- [feat] Implemented partitioned filters
- [feat] Allow calling bulk ingestion on non-empty keyspaces
- [feat] Introduced level-based configuration policies for most configuration parameters
- [feat] Journal compression for large values
- [feat] Database locking using the new Rust file locking API
- [feat] Rewritten key-value separation to run during compactions, instead of dedicated GC runs
- [feat] Full file checksums to allow fast database corruption checks (in the future)
- [feat] Checksum check on block & blob reads
- [api] Make Ingestion API more flexible
- [feat] Shortening eligible sequence numbers when compacting into the last level to save disk space
- [api] Change constructor to `Database::builder` instead of `Config::new`
- [api] Changed naming of keyspace->database, and partition->keyspace
- [api] Change transaction feature flags to be separate structs, `OptimisticTxDatabase` and `SingleWriterTxDatabase`
- [api] Changed snapshot error type, fixes #156
- [api] Unified transactions read operations and snapshots with `Readable` trait
- [api] Guard API for iterator values
- [api] Removed old garbage collection APIs
- [api] `metrics` feature flag for cache hit rates etc. (will be exposed in the future)
- [api] Change `bytes` feature flag to `bytes_1` to pin its version
- [api] Make read operations in optimistic write transactions non-mut
- [fix] Consider blob files in FIFO compaction size limit, fixes #133
- [perf] Use a single hash per key for filters, instead of two
- [perf] Improve leveled compaction scoring
- [perf] Improve leveled compaction picking to use less hashing and heap allocations
- [perf] Use `quick-cache` for file descriptor caching
- [perf] Promote levels immediately to L6 to get rid of tombstones easily
- [perf] Rewritten maintenance task bookkeeping, and write stalling mechanisms to be less aggressive
- [perf] Allow `lsm-tree` flushes to merge multiple sealed memtables into L0, if necessary
- [perf] Skip heap allocation in blob memtable inserts
- [perf] Skip compression when rewriting compressed blob files
- [msrv] Increased MSRV to **1.91**
- [misc] Blob file descriptor caching
- [misc] Use Rust native `path::absolute`, removing `path-absolutize` dependency
- [misc] Remove `std-semaphore` dependency
- [misc] Remove `miniz` (will be replaced in the future)
- [misc] Use `byteorder-lite` as drop-in replacement for `byteorder`
- [refactor] Changed background workers to be a single thread pool
- [internal] Store keyspace configurations in a meta keyspace, instead of individual binary config files
- [internal] Use `sfa` for most file scaffolding in `lsm-tree`
crates/fjall/CONTRIBUTING.md
-11
View File
@@ -1,11 +0,0 @@
# Contributing
## License
By contributing to this project, you agree that your contributions will be licensed under the project's license (MIT OR Apache-2.0).
Thank you for your contribution!
## Looking for issues?
https://github.com/fjall-rs/fjall/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
crates/fjall/Cargo.toml
+4 -14
View File
@@ -5,11 +5,8 @@ license.workspace = true
version.workspace = true
edition.workspace = true
readme.workspace = true
include = ["src/**/*", "LICENSE-APACHE", "LICENSE-MIT", "README.md"]
repository.workspace = true
homepage.workspace = true
keywords = ["database", "key-value", "lsm", "rocksdb", "leveldb"]
categories = ["data-structures", "database-implementations", "algorithms"]
[lib]
name = "fjall"
@@ -18,9 +15,6 @@ path = "src/lib.rs"
[features]
default = ["lz4"]
lz4 = ["lsm-tree/lz4", "dep:lz4_flex"]
bytes_1 = ["lsm-tree/bytes_1"]
metrics = ["lsm-tree/metrics"]
__internal_whitebox = []
[dependencies]
byteorder = { package = "byteorder-lite", version = "0.1.0" }
@@ -28,15 +22,11 @@ byteview = { workspace = true }
lsm-tree = { workspace = true, default-features = false, features = [] }
log = { workspace = true }
tempfile = { workspace = true }
dashmap = "6.1.0"
xxhash-rust = { version = "0.8.15", features = ["xxh3"] }
dashmap = "6.2.1"
xxhash-rust = { version = "0.8.18", features = ["xxh3"] }
lz4_flex = { workspace = true, features = ["checked-decode"], optional = true }
flume = { version = "0.12.0", default-features = false }
[dev-dependencies]
nanoid = "0.4.0"
test-log = "0.2.18"
rand = "0.10.0"
[package.metadata.cargo-all-features]
denylist = ["__internal_whitebox"]
nanoid = "0.5.0"
test-log = "0.2.21"
crates/fjall/README.md
-207
View File
@@ -1,207 +0,0 @@
<p align="center">
<img src="/kawaii.png" height="200">
</p>
<p align="center">
<a href="https://github.com/fjall-rs/fjall/actions/workflows/test.yml">
<img src="https://github.com/fjall-rs/fjall/actions/workflows/test.yml/badge.svg" alt="CI" />
</a>
<a href="https://docs.rs/fjall">
<img src="https://img.shields.io/docsrs/fjall?color=green" alt="docs.rs" />
</a>
<a href="https://crates.io/crates/fjall">
<img src="https://img.shields.io/crates/v/fjall?color=blue" alt="Crates.io" />
</a>
<img src="https://img.shields.io/badge/MSRV-1.90.0-blue" alt="MSRV" />
<a href="https://deps.rs/repo/github/fjall-rs/fjall">
<img src="https://deps.rs/repo/github/fjall-rs/fjall/status.svg" alt="dependency status" />
</a>
</p>
<p align="center">
<a href="https://discord.com/invite/HvYGp4NFFk">
<img src="https://img.shields.io/discord/1240426554111164486" alt="Discord" />
</a>
<a href="https://bsky.app/profile/fjallrs.bsky.social">
<img src="https://img.shields.io/badge/bluesky-blue" alt="Bluesky" />
</a>
</p>
*Fjall* _(Nordic: "Mountain")_ is a log-structured, embeddable key-value storage engine written in Rust.
It features:
- A thread-safe BTreeMap-like API
- 100% safe & stable Rust
- LSM-tree-based storage similar to `RocksDB`
- Range & prefix searching with forward and reverse iteration
- Multiple keyspaces (a.k.a. column families) with cross-keyspace atomic semantics
- Built-in compression (default = `LZ4`)
- Serializable transactions (optional)
- Key-value separation for large blob use cases (optional)
- Custom compaction filters to run custom logic during compactions (optional)
- Automatic background maintenance
It is not:
- A standalone database server
- A relational or wide-column database: it has no built-in notion of columns or query language
## Sponsors
<a href="https://sqlsync.dev">
<picture>
<source width="240" alt="Orbitinghail" media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/fjall-rs/fjall-rs.github.io/d22fcb1e6966ce08327ea3bf6cf2ea86a840b071/public/logos/orbitinghail.svg" />
<source width="240" alt="Orbitinghail" media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/fjall-rs/fjall-rs.github.io/d22fcb1e6966ce08327ea3bf6cf2ea86a840b071/public/logos/orbitinghail_dark.svg" />
<img width="240" alt="Orbitinghail" src="https://raw.githubusercontent.com/fjall-rs/fjall-rs.github.io/d22fcb1e6966ce08327ea3bf6cf2ea86a840b071/public/logos/orbitinghail_dark.svg" />
</picture>
</a>
## Basic usage
```bash
cargo add fjall
```
```rust
use fjall::{Database, KeyspaceCreateOptions, PersistMode};
fn main() -> fjall::Result<()> {
// A database may contain multiple keyspaces
// You should probably only use a single database for your application
let db = Database::builder(path).open()?;
// TxDatabase::builder for transactional semantics
// Each keyspace is its own physical LSM-tree, and thus isolated from other keyspaces
let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?;
// Write some data
items.insert("a", "hello")?;
// And retrieve it
let bytes = items.get("a")?;
// Or remove it again
items.remove("a")?;
// Search by prefix
for kv in items.prefix("prefix") {
// ...
}
// Search by range
for kv in items.range("a"..="z") {
// ...
}
// Iterators implement DoubleEndedIterator, so you can search backwards, too!
for kv in items.prefix("prefix").rev() {
// ...
}
// Sync the journal to disk to make sure data is definitely durable
// When the database is dropped, it will try to persist with `PersistMode::SyncAll` automatically
db.persist(PersistMode::SyncAll)
}
```
> [!TIP]
> Like any typical key-value store, keys are stored in lexicographic order.
> If you are storing integer keys (e.g. timeseries data), you should use the big endian form to have predictable ordering.
## Durability
To support different kinds of workloads, Fjall is agnostic about the type of durability
your application needs.
After writing data (`insert`, `remove` or committing a write batch/transaction), you can choose to call [`Database::persist`](https://docs.rs/fjall/latest/fjall/struct.Database.html#method.persist) which takes a [`PersistMode`](https://docs.rs/fjall/latest/fjall/enum.PersistMode.html) parameter.
By default, any operation will flush to OS buffers, but **not** to disk.
This matches RocksDB's default durability.
Also, when dropped, the database will try to persist the journal *to disk* synchronously.
## Multithreading, Async and Multiprocess
> [!WARNING]
> A single database may **not** be loaded in parallel from separate *processes*.
Fjall is internally synchronized for multi-*threaded* access, so you can clone around the `Database` and `Keyspace`s as needed, without needing to lock yourself.
For an async example, see the [`tokio`](https://github.com/fjall-rs/fjall/tree/main/examples/tokio) example.
## Memory usage
Generally, memory for loaded data, indexes etc. is managed on a per-block basis, and capped by the block cache capacity.
Note that this also applies to returned values: When you hold a `Slice`, it keeps the backing buffer alive (which may be a block).
If you know that you are going to keep a value around for a long time, you may want to copy it out into a new `Vec<u8>`, `Box<[u8]>`, `Arc<[u8]>` or new `Slice` (using `Slice::new`).
> [!NOTE]
> It is recommended to configure the block cache capacity to be ~20-25% of the available memory - or more **if** the data set fits _fully_ into memory.
Additionally, orthogonally to the block cache, each `Keyspace` has its own write buffer (["Memtable"](https://docs.rs/fjall/latest/fjall/struct.KeyspaceCreateOptions.html#method.max_memtable_size)) which is the unit of data flushed back into the "proper" index structure.
## Error handling
Fjall returns an [error enum](https://docs.rs/fjall/latest/fjall/enum.Error.html), however these variants are mostly used for debugging and tracing purposes, so your application is not expected to handle specific errors.
It's best to let the application crash and restart, which is the [safest way to recover from transient I/O errors](https://ramalagappan.github.io/pdfs/papers/cuttlefs.pdf).
## Transactional modes
The backing store (`lsm-tree`) is a MVCC key-value store, allowing repeatable snapshot reads.
However this isolation level can not do read-modify-write operations without the chance of lost updates.
Also, `WriteBatch` does not allow reading the intermediary state back as you would expect from a proper transaction.
For that reason, if you need transactional semantics, you need to use one of the transactional database implementation (`OptimisticTxDatabase` or `SingleWriterTxDatabase`).
TL;DR: Fjall supports both transactional and non-transactional workloads.
Chances are you want to use a transactional database, unless you know your workload does not need serializable transaction semantics.
### Single writer
Opens a transactional database for single-writer (serialized) transactions.
Single writer means only a single **write** transaction can run at a time.
This is trivially serializable because it _literally_ serializes write transactions.
### Optimistic
Opens a transactional database for multi-writer, serializable transactions.
Conflict checking is done using optimistic concurrency control, meaning transactions can conflict and may have to be rerun.
## Feature flags
### lz4
Allows using `LZ4` compression, powered by [`lz4_flex`](https://github.com/PSeitz/lz4_flex).
*Enabled by default.*
### bytes_1
Uses [`bytes`](https://github.com/tokio-rs/bytes) 1.x as the underlying `Slice` type.
Otherwise, [`byteview`](https://github.com/fjall-rs/byteview) is used instead.
*Disabled by default.*
## Stable disk format
Future breaking changes will result in a major version bump and a migration path.
For the underlying LSM-tree implementation, see: <https://crates.io/crates/lsm-tree>.
## Examples
[See here](https://github.com/fjall-rs/fjall/tree/main/examples) for practical examples.
## Contributing
How can you help?
- [Ask a question](https://github.com/fjall-rs/fjall/discussions/new?category=q-a)
- or join the Discord server: [https://discord.com/invite/HvYGp4NFFk](https://discord.com/invite/HvYGp4NFFk)
- [Post benchmarks and things you created](https://github.com/fjall-rs/fjall/discussions/new?category=show-and-tell)
- [Open a PR](https://github.com/fjall-rs/fjall/compare),
- [See open issues to pick up here](https://github.com/search?q=org%3Afjall-rs+label%3A%22help+wanted%22+state%3Aopen+&type=issues)
- [Open an issue](https://github.com/fjall-rs/fjall/issues/new) (bug report, weirdness)
## License
All source code is licensed under MIT OR Apache-2.0.
All contributions are to be licensed as MIT OR Apache-2.0.
crates/fjall/commit.nu
-64
View File
@@ -1,64 +0,0 @@
let machines = [
# Fly.io performance
# "fly.performance.1x",
# "fly.performance.2x",
"fly.performance.4x",
# "fly.performance.8x",
# "fly.performance.16x",
# EC2 T2
# "aws.ec2.t2.nano",
# "aws.ec2.t2.micro",
# "aws.ec2.t2.small",
# "aws.ec2.t2.medium",
# "aws.ec2.t2.large",
# "aws.ec2.t2.xlarge",
# "aws.ec2.t2.2xlarge",
# EC2 T3
# "aws.ec2.t3.nano",
# "aws.ec2.t3.micro",
# "aws.ec2.t3.small",
"aws.ec2.t3.medium",
# "aws.ec2.t3.large",
# "aws.ec2.t3.xlarge",
# "aws.ec2.t3.2xlarge",
# EC2 T3a
# "aws.ec2.t3a.nano",
# "aws.ec2.t3a.micro",
# "aws.ec2.t3a.small",
# "aws.ec2.t3a.medium",
# "aws.ec2.t3a.large",
# "aws.ec2.t3a.xlarge",
# "aws.ec2.t3a.2xlarge",
# EC2 T4g
# "aws.ec2.t4g.nano",
# "aws.ec2.t4g.micro",
# "aws.ec2.t4g.small",
# "aws.ec2.t4g.medium",
# "aws.ec2.t4g.large",
# "aws.ec2.t4g.xlarge",
# "aws.ec2.t4g.2xlarge",
# EC2 M4
# "aws.ec2.m4.large",
]
let table = $env.TABLE_NAME
let commit = $env.COMMIT
print $"Queuing ($commit)"
for machine in $machines {
let q_pk = $"q#($machine)"
print $"Adding queue item for ($machine)"
let item = {
pk: { S: $q_pk },
sk: { S: $commit },
version: { S: "2" },
}
aws dynamodb put-item --table-name $table --item ($item | to json)
}
crates/fjall/compile_examples.mjs
-60
View File
@@ -1,60 +0,0 @@
import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
import { readdir } from "node:fs/promises";
import { resolve } from "node:path";
const examplesFolder = "examples";
for (const exampleName of await readdir(examplesFolder)) {
const folder = resolve(examplesFolder, exampleName);
{
console.error(`Testing ${exampleName}`);
const proc = spawn("cargo test", {
cwd: folder,
shell: true,
});
proc.stdout.on("data", buf => console.log(String(buf)));
proc.stderr.on("data", buf => console.error(String(buf)));
await new Promise((resolve, _) => {
proc.on("exit", () => {
if (proc.exitCode > 0) {
console.error(`${exampleName} FAILED`);
process.exit(1);
}
else {
resolve();
}
})
});
}
if (existsSync(resolve(folder, ".run"))) {
console.error(`Running ${exampleName}`);
const proc = spawn("cargo run", {
cwd: folder,
shell: true,
});
proc.stdout.on("data", buf => console.log(String(buf)));
proc.stderr.on("data", buf => console.error(String(buf)));
await new Promise((resolve, _) => {
proc.on("exit", () => {
if (proc.exitCode > 0) {
console.error(`${exampleName} FAILED`);
process.exit(1);
}
else {
resolve();
}
})
});
}
console.error(`${exampleName} OK`);
}
crates/fjall/kawaii.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 90 KiB

crates/fjall/logo.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 18 KiB

crates/fjall/renovate.json
-6
View File
@@ -1,6 +0,0 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:base"
]
}
crates/fjall/src/batch/item.rs
-1
View File
@@ -35,7 +35,6 @@ impl std::fmt::Debug for Item {
ValueType::Value => "V",
ValueType::Tombstone => "T",
ValueType::WeakTombstone => "W",
ValueType::Indirection => "Vb",
},
self.value
)
crates/fjall/src/batch/mod.rs
-1
View File
@@ -148,7 +148,6 @@ impl WriteBatch {
ValueType::Value => item.keyspace.tree.insert(item.key, item.value, batch_seqno),
ValueType::Tombstone => item.keyspace.tree.remove(item.key, batch_seqno),
ValueType::WeakTombstone => item.keyspace.tree.remove_weak(item.key, batch_seqno),
ValueType::Indirection => unreachable!(),
};
batch_size += item_size;
crates/fjall/src/builder.rs
+6 -8
View File
@@ -2,21 +2,19 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{db_config::CompactionFilterAssigner, tx::single_writer::Openable, Config};
use crate::{Config, Database, db_config::CompactionFilterAssigner};
use lsm_tree::{Cache, CompressionType, DescriptorTable};
use std::{marker::PhantomData, path::Path, sync::Arc};
use std::{path::Path, sync::Arc};
/// Database builder
pub struct Builder<O: Openable> {
pub struct Builder {
inner: Config,
_phantom: PhantomData<O>,
}
impl<O: Openable> Builder<O> {
impl Builder {
pub(crate) fn new(path: &Path) -> Self {
Self {
inner: Config::new(path),
_phantom: PhantomData,
}
}
@@ -31,8 +29,8 @@ impl<O: Openable> Builder<O> {
/// # Errors
///
/// Errors if an I/O error occurred, or if the database can not be opened.
pub fn open(self) -> crate::Result<O> {
O::open(self.inner)
pub fn open(self) -> crate::Result<Database> {
Database::open(self.inner)
}
/// Sets the cache capacity in bytes.
crates/fjall/src/compaction/mod.rs
+1 -1
View File
@@ -4,7 +4,7 @@
pub(crate) mod worker;
pub use lsm_tree::compaction::{Fifo, Leveled, Levelled};
pub use lsm_tree::compaction::{Leveled, Levelled};
/// Compaction filter utilities
pub mod filter {
crates/fjall/src/compaction/worker.rs
+3 -3
View File
@@ -2,7 +2,7 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{snapshot_tracker::SnapshotTracker, stats::Stats, Keyspace};
use crate::{Keyspace, snapshot_tracker::SnapshotTracker, stats::Stats};
use lsm_tree::AbstractTree;
use std::time::Instant;
@@ -23,7 +23,7 @@ pub fn run(
keyspace.name,
);
let strategy = keyspace.config.compaction_strategy.clone();
let strategy = std::sync::Arc::new(crate::compaction::Leveled::default());
stats.active_compaction_count.fetch_add(1, Relaxed);
@@ -33,7 +33,7 @@ pub fn run(
if let Err(e) = keyspace
.tree
.compact(strategy.clone(), snapshot_tracker.get_seqno_safe_to_gc())
.compact(strategy, snapshot_tracker.get_seqno_safe_to_gc())
{
log::error!("Compaction failed: {e:?}");
stats.active_compaction_count.fetch_sub(1, Relaxed);
crates/fjall/src/db.rs
+9 -31
View File
@@ -3,12 +3,13 @@
// (found in the LICENSE-* files in the repository)
use crate::{
HashMap, Keyspace, KeyspaceCreateOptions,
batch::WriteBatch,
db_config::Config,
file::{fsync_directory, KEYSPACES_FOLDER, LOCK_FILE, VERSION_MARKER},
file::{KEYSPACES_FOLDER, LOCK_FILE, VERSION_MARKER, fsync_directory},
flush::manager::FlushManager,
journal::{manager::JournalManager, writer::PersistMode, Journal},
keyspace::{name::is_valid_keyspace_name, KeyspaceKey},
journal::{Journal, manager::JournalManager, writer::PersistMode},
keyspace::{KeyspaceKey, name::is_valid_keyspace_name},
locked_file::LockedFileGuard,
meta_keyspace::MetaKeyspace,
poison::{PoisonDart, PoisonSignal},
@@ -17,17 +18,15 @@ use crate::{
snapshot_tracker::SnapshotTracker,
stats::Stats,
supervisor::{Supervisor, SupervisorInner},
tx::single_writer::Openable,
version::FormatVersion,
worker_pool::{WorkerMessage, WorkerPool},
write_buffer_manager::WriteBufferManager,
HashMap, Keyspace, KeyspaceCreateOptions,
};
use lsm_tree::{AbstractTree, SequenceNumberCounter};
use std::{
fs::remove_dir_all,
path::Path,
sync::{atomic::AtomicUsize, Arc, RwLock},
sync::{Arc, RwLock, atomic::AtomicUsize},
};
pub type Keyspaces = HashMap<KeyspaceKey, Keyspace>;
@@ -106,9 +105,6 @@ impl Drop for DatabaseInner {
);
}
}
#[cfg(feature = "__internal_whitebox")]
crate::drop::decrement_drop_counter();
}
}
@@ -131,15 +127,6 @@ impl std::ops::Deref for Database {
}
}
impl Openable for Database {
fn open(config: Config) -> crate::Result<Self>
where
Self: Sized,
{
Self::open(config)
}
}
impl Database {
/// Opens a cross-keyspace snapshot.
///
@@ -152,7 +139,7 @@ impl Database {
}
/// Creates a new database builder to create or open a database at `path`.
pub fn builder(path: impl AsRef<Path>) -> crate::DatabaseBuilder<Self> {
pub fn builder(path: impl AsRef<Path>) -> crate::DatabaseBuilder {
crate::DatabaseBuilder::new(path.as_ref())
}
@@ -391,9 +378,6 @@ impl Database {
let db = Self::create_or_recover(config)?;
// db.start_background_threads()?;
#[cfg(feature = "__internal_whitebox")]
crate::drop::increment_drop_counter();
Ok(db)
}
@@ -477,9 +461,6 @@ impl Database {
self.meta_keyspace
.create_keyspace(keyspace_id, &name, handle.clone(), keyspaces)?;
#[cfg(feature = "__internal_whitebox")]
crate::drop::increment_drop_counter();
handle
})
}
@@ -549,12 +530,12 @@ impl Database {
"It looks like you are trying to open a V2 database - the database needs a manual migration, a tool is available at https://github.com/fjall-rs/migrate-v2-v3."
);
}
if version as u8 > 4 {
if version as u8 > 5 {
log::error!(
"It looks like you are trying to open a database from the future. Are you a time traveller?"
);
}
if version != FormatVersion::V4 {
if version != FormatVersion::V5 {
return Err(crate::Error::InvalidVersion(Some(version)));
}
} else {
@@ -718,9 +699,6 @@ impl Database {
lsm_tree::ValueType::WeakTombstone => {
tree.remove_weak(item.key, batch.seqno);
}
lsm_tree::ValueType::Indirection => {
unreachable!()
}
}
}
@@ -837,7 +815,7 @@ impl Database {
// NOTE: Lastly, fsync version marker, which contains the version
let mut marker = std::fs::File::create_new(config.path.join(VERSION_MARKER))?;
FormatVersion::V4.write_file_header(&mut marker)?;
FormatVersion::V5.write_file_header(&mut marker)?;
marker.sync_all()?;
// IMPORTANT: fsync folders on Unix
crates/fjall/src/db_test.rs
+2 -106
View File
@@ -1,4 +1,4 @@
use crate::{Database, KeyspaceCreateOptions, KvSeparationOptions};
use crate::{Database, KeyspaceCreateOptions};
use test_log::test;
#[test_log::test]
@@ -39,83 +39,6 @@ fn clear_recover_sealed() -> crate::Result<()> {
Ok(())
}
// TODO: investigate: flaky on macOS???
#[cfg(feature = "__internal_whitebox")]
#[test]
#[ignore = "restore"]
fn whitebox_db_drop() -> crate::Result<()> {
use crate::Database;
{
let folder = tempfile::tempdir()?;
assert_eq!(0, crate::drop::load_drop_counter());
let db = Database::builder(&folder).open()?;
assert_eq!(5, crate::drop::load_drop_counter());
drop(db);
assert_eq!(0, crate::drop::load_drop_counter());
}
{
let folder = tempfile::tempdir()?;
assert_eq!(0, crate::drop::load_drop_counter());
let db = Database::builder(&folder).open()?;
assert_eq!(5, crate::drop::load_drop_counter());
let tree = db.keyspace("default", Default::default)?;
assert_eq!(6, crate::drop::load_drop_counter());
drop(tree);
drop(db);
assert_eq!(0, crate::drop::load_drop_counter());
}
{
let folder = tempfile::tempdir()?;
assert_eq!(0, crate::drop::load_drop_counter());
let db = Database::builder(&folder).open()?;
assert_eq!(5, crate::drop::load_drop_counter());
let _tree = db.keyspace("default", Default::default)?;
assert_eq!(6, crate::drop::load_drop_counter());
let _tree2 = db.keyspace("different", Default::default)?;
assert_eq!(7, crate::drop::load_drop_counter());
}
assert_eq!(0, crate::drop::load_drop_counter());
Ok(())
}
#[cfg(feature = "__internal_whitebox")]
#[test]
#[ignore = "restore"]
fn whitebox_db_drop_2() -> crate::Result<()> {
use crate::{Database, KeyspaceCreateOptions};
let folder = tempfile::tempdir()?;
{
let db = Database::builder(&folder).open()?;
let tree = db.keyspace("tree", KeyspaceCreateOptions::default)?;
let tree2 = db.keyspace("tree1", KeyspaceCreateOptions::default)?;
tree.insert("a", "a")?;
tree2.insert("b", "b")?;
tree.rotate_memtable_and_wait()?;
}
assert_eq!(0, crate::drop::load_drop_counter());
Ok(())
}
#[test]
pub fn test_exotic_keyspace_names() -> crate::Result<()> {
let folder = tempfile::tempdir()?;
@@ -184,31 +107,6 @@ fn recover_sealed_order() -> crate::Result<()> {
Ok(())
}
#[test]
#[expect(clippy::unwrap_used)]
fn recover_sealed_blob() -> crate::Result<()> {
let folder = tempfile::tempdir()?;
for i in 0_u128..3 {
let db = Database::create_or_recover(Database::builder(folder.path()).into_config())?;
let tree = db.keyspace("default", || {
KeyspaceCreateOptions::default()
.max_memtable_size(1_000)
.with_kv_separation(Some(KvSeparationOptions::default()))
})?;
assert_eq!(i, tree.len()?.try_into().unwrap());
tree.insert(i.to_be_bytes(), i.to_be_bytes().repeat(1_024))?;
assert_eq!(i + 1, tree.len()?.try_into().unwrap());
tree.rotate_memtable_and_wait()?;
}
Ok(())
}
#[test]
#[expect(clippy::unwrap_used)]
fn recover_sealed_pair_1() -> crate::Result<()> {
@@ -221,9 +119,7 @@ fn recover_sealed_pair_1() -> crate::Result<()> {
KeyspaceCreateOptions::default().max_memtable_size(1_000)
})?;
let tree2 = db.keyspace("default2", || {
KeyspaceCreateOptions::default()
.max_memtable_size(1_000)
.with_kv_separation(Some(KvSeparationOptions::default()))
KeyspaceCreateOptions::default().max_memtable_size(1_000)
})?;
assert_eq!(i, tree.len()?.try_into().unwrap());
crates/fjall/src/drop.rs
-25
View File
@@ -1,25 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use std::sync::atomic::Ordering::Relaxed;
use std::sync::{atomic::AtomicUsize, OnceLock};
static DROP_COUNTER: OnceLock<AtomicUsize> = OnceLock::new();
pub fn increment_drop_counter() {
get_drop_counter().fetch_add(1, Relaxed);
}
pub fn decrement_drop_counter() {
get_drop_counter().fetch_sub(1, Relaxed);
}
pub fn get_drop_counter<'a>() -> &'a AtomicUsize {
DROP_COUNTER.get_or_init(AtomicUsize::default)
}
#[must_use]
pub fn load_drop_counter() -> usize {
get_drop_counter().load(Relaxed)
}
crates/fjall/src/error.rs
+1 -1
View File
@@ -3,7 +3,7 @@
// (found in the LICENSE-* files in the repository)
use crate::{
journal::error::RecoveryError as JournalRecoveryError, version::FormatVersion, CompressionType,
CompressionType, journal::error::RecoveryError as JournalRecoveryError, version::FormatVersion,
};
/// Errors that may occur in the storage engine
crates/fjall/src/ingestion.rs
+17 -29
View File
@@ -2,17 +2,17 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{worker_pool::WorkerMessage, Keyspace};
use lsm_tree::{AnyIngestion, UserKey, UserValue};
use crate::{Keyspace, worker_pool::WorkerMessage};
use lsm_tree::{Ingestion as TreeIngestion, UserKey, UserValue};
pub struct Ingestion<'a> {
keyspace: &'a Keyspace,
inner: AnyIngestion<'a>,
inner: TreeIngestion<'a>,
}
impl<'a> Ingestion<'a> {
pub fn new(keyspace: &'a Keyspace) -> crate::Result<Self> {
let inner = keyspace.tree.ingestion()?;
let inner = TreeIngestion::new(&keyspace.tree)?;
Ok(Self { keyspace, inner })
}
@@ -20,37 +20,20 @@ impl<'a> Ingestion<'a> {
&mut self,
key: K,
value: V,
) -> crate::Result<()> {
self.inner.write(key, value).map_err(Into::into)
}
#[doc(hidden)]
pub fn write_prevalidated<K: Into<UserKey>, V: Into<UserValue>>(
&mut self,
key: K,
value: V,
) -> crate::Result<()> {
self.inner
.write_prevalidated(key, value)
.write(key.into(), value.into())
.map_err(Into::into)
}
pub fn write_tombstone<K: Into<UserKey>>(&mut self, key: K) -> crate::Result<()> {
self.inner.write_tombstone(key).map_err(Into::into)
self.inner.write_tombstone(key.into()).map_err(Into::into)
}
#[doc(hidden)]
pub fn write_weak_tombstone<K: Into<UserKey>>(&mut self, key: K) -> crate::Result<()> {
self.inner.write_weak_tombstone(key).map_err(Into::into)
}
#[doc(hidden)]
pub fn write_prevalidated_weak_tombstone<K: Into<UserKey>>(
&mut self,
key: K,
) -> crate::Result<()> {
self.inner
.write_prevalidated_weak_tombstone(key)
.write_weak_tombstone(key.into())
.map_err(Into::into)
}
@@ -72,7 +55,7 @@ impl<'a> Ingestion<'a> {
// insert seqno=1
let _journal_lock = self.keyspace.supervisor.journal.get_writer();
self.finish_inner()
self.finish_inner(false)
}
/// Finishes the ingestion without taking the global journal writer lock.
@@ -81,12 +64,17 @@ impl<'a> Ingestion<'a> {
/// the same database. Exclusive ingestions into independent keyspaces may
/// still finish concurrently.
pub fn finish_exclusive(self) -> crate::Result<()> {
self.finish_inner()
self.finish_inner(true)
}
fn finish_inner(self) -> crate::Result<()> {
self.inner
.finish()
fn finish_inner(self, exclusive: bool) -> crate::Result<()> {
let result = if exclusive {
self.inner.finish_exclusive()
} else {
self.inner.finish()
};
result
.inspect(|()| {
self.keyspace
.worker_messager
crates/fjall/src/iter.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{snapshot_nonce::SnapshotNonce, Guard};
use crate::{Guard, snapshot_nonce::SnapshotNonce};
type InnerIter = Box<dyn DoubleEndedIterator<Item = lsm_tree::IterGuardImpl> + Send + 'static>;
crates/fjall/src/journal/batch_reader.rs
+7 -3
View File
@@ -3,7 +3,7 @@
// (found in the LICENSE-* files in the repository)
use super::reader::JournalReader;
use crate::{journal::entry::Entry, keyspace::InternalKeyspaceId, JournalRecoveryError};
use crate::{JournalRecoveryError, journal::entry::Entry, keyspace::InternalKeyspaceId};
use lsm_tree::{SeqNo, UserKey, UserValue, ValueType};
use std::{fs::OpenOptions, hash::Hasher};
@@ -63,7 +63,9 @@ impl JournalBatchReader {
fn on_close(&self) -> crate::Result<()> {
if self.is_in_batch {
log::debug!("Invalid batch: missing terminator, but last batch, so probably incomplete, discarding to keep atomicity");
log::debug!(
"Invalid batch: missing terminator, but last batch, so probably incomplete, discarding to keep atomicity"
);
// Discard batch
self.truncate_to(self.last_valid_pos)?;
@@ -125,7 +127,9 @@ impl Iterator for JournalBatchReader {
self.checksum_builder = xxhash_rust::xxh3::Xxh3::new();
if got_checksum != expected_checksum {
log::error!("Invalid batch: checksum check failed, expected: {expected_checksum}, got: {got_checksum}");
log::error!(
"Invalid batch: checksum check failed, expected: {expected_checksum}, got: {got_checksum}"
);
return Some(Err(JournalRecovery(JournalRecoveryError::ChecksumMismatch)));
}
crates/fjall/src/journal/entry.rs
+2 -2
View File
@@ -2,11 +2,11 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{file::MAGIC_BYTES, keyspace::InternalKeyspaceId, Slice};
use crate::{Slice, file::MAGIC_BYTES, keyspace::InternalKeyspaceId};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use lsm_tree::{
coding::{Decode, Encode},
CompressionType, SeqNo, UserKey, UserValue, ValueType,
coding::{Decode, Encode},
};
use std::io::{Read, Write};
crates/fjall/src/journal/manager.rs
-6
View File
@@ -49,17 +49,11 @@ pub struct JournalManager {
impl Drop for JournalManager {
fn drop(&mut self) {
log::trace!("Dropping journal manager");
#[cfg(feature = "__internal_whitebox")]
crate::drop::decrement_drop_counter();
}
}
impl JournalManager {
pub(crate) fn new() -> Self {
#[cfg(feature = "__internal_whitebox")]
crate::drop::increment_drop_counter();
Self {
items: Vec::with_capacity(10),
disk_space_in_bytes: 0,
crates/fjall/src/journal/mod.rs
+1 -7
View File
@@ -18,7 +18,7 @@ use crate::file::fsync_directory;
use batch_reader::JournalBatchReader;
use lsm_tree::CompressionType;
use reader::JournalReader;
use recovery::{recover_journals, RecoveryResult};
use recovery::{RecoveryResult, recover_journals};
use std::{
path::{Path, PathBuf},
sync::{Mutex, MutexGuard},
@@ -53,9 +53,6 @@ impl Drop for Journal {
log::error!("Flush error on drop: {e:?}");
}
}
#[cfg(feature = "__internal_whitebox")]
crate::drop::decrement_drop_counter();
}
}
@@ -92,9 +89,6 @@ impl Journal {
// IMPORTANT: fsync folder on Unix
fsync_directory(folder)?;
#[cfg(feature = "__internal_whitebox")]
crate::drop::increment_drop_counter();
Ok(Self {
writer: Mutex::new(writer),
})
crates/fjall/src/journal/writer.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use super::entry::{serialize_marker_item, Entry};
use super::entry::{Entry, serialize_marker_item};
use crate::{
batch::item::Item as BatchItem, file::fsync_directory, journal::recovery::JournalId,
keyspace::InternalKeyspaceId,
crates/fjall/src/keyspace/config/compression.rs
+1 -1
View File
@@ -5,8 +5,8 @@
use crate::keyspace::config::{DecodeConfig, EncodeConfig};
use byteorder::{ReadBytesExt, WriteBytesExt};
use lsm_tree::{
coding::{Decode, Encode},
CompressionType,
coding::{Decode, Encode},
};
impl EncodeConfig for crate::config::CompressionPolicy {
crates/fjall/src/keyspace/mod.rs
+9 -53
View File
@@ -11,6 +11,7 @@ mod write_delay;
mod test;
use crate::{
Database, Guard, Iter,
file::{KEYSPACES_FOLDER, LSM_CURRENT_VERSION_MARKER},
flush::Task as FlushTask,
ingestion::Ingestion,
@@ -18,14 +19,13 @@ use crate::{
poison::PoisonSignal,
supervisor::Supervisor,
worker_pool::WorkerMessage,
Database, Guard, Iter,
};
use lsm_tree::{AbstractTree, AnyTree, SeqNo, UserKey, UserValue};
use lsm_tree::{AbstractTree, SeqNo, Tree, UserKey, UserValue};
use options::CreateOptions;
use std::{
ops::RangeBounds,
path::Path,
sync::{atomic::AtomicBool, Arc, MutexGuard},
sync::{Arc, MutexGuard, atomic::AtomicBool},
time::Duration,
};
use write_delay::perform_write_stall;
@@ -50,7 +50,6 @@ pub fn apply_to_base_config(
.index_block_pinning_policy(our_config.index_block_pinning_policy.clone())
.data_block_hash_ratio_policy(our_config.data_block_hash_ratio_policy.clone())
.expect_point_read_hits(our_config.expect_point_read_hits)
.with_kv_separation(our_config.kv_separation_opts.clone())
.index_block_partitioning_policy(our_config.index_block_partitioning_policy.clone())
.filter_block_partitioning_policy(our_config.filter_block_partitioning_policy.clone())
.filter_policy(our_config.filter_policy.clone())
@@ -78,7 +77,7 @@ pub struct KeyspaceInner {
/// LSM-tree wrapper
#[doc(hidden)]
pub tree: AnyTree,
pub tree: Tree,
pub(crate) supervisor: Supervisor,
@@ -132,9 +131,6 @@ impl Drop for KeyspaceInner {
}
}
}
#[cfg(feature = "__internal_whitebox")]
crate::drop::decrement_drop_counter();
}
}
@@ -183,14 +179,6 @@ impl std::hash::Hash for Keyspace {
}
impl Keyspace {
#[inline]
fn standard_tree(&self) -> &lsm_tree::Tree {
let lsm_tree::AnyTree::Standard(tree) = &self.tree else {
panic!("standard keyspace operation used with a blob keyspace");
};
tree
}
#[doc(hidden)]
#[must_use]
pub fn id(&self) -> InternalKeyspaceId {
@@ -271,14 +259,6 @@ impl Keyspace {
Ok(())
}
/// Returns the number of blob bytes on disk that are not referenced.
///
/// These will be reclaimed over time by blob garbage collection automatically.
#[must_use]
pub fn fragmented_blob_bytes(&self) -> u64 {
self.tree.stale_blob_bytes()
}
#[doc(hidden)]
#[must_use]
pub fn sealed_memtable_count(&self) -> usize {
@@ -304,7 +284,7 @@ impl Keyspace {
pub(crate) fn from_database(
keyspace_id: InternalKeyspaceId,
db: &Database,
tree: AnyTree,
tree: Tree,
name: KeyspaceKey,
config: CreateOptions,
) -> Self {
@@ -362,17 +342,6 @@ impl Keyspace {
})))
}
/// Returns the metrics struct of the underlying LSM-tree.
///
/// # Note
///
/// This function is experimental and metric names may change in future releases.
#[cfg(feature = "metrics")]
#[doc(hidden)]
pub fn metrics(&self) -> &lsm_tree::Metrics {
&**self.tree.metrics()
}
/// Returns the underlying LSM-tree's path.
#[must_use]
pub fn path(&self) -> &Path {
@@ -432,7 +401,7 @@ impl Keyspace {
) -> impl DoubleEndedIterator<Item = crate::Result<crate::KvPair>> + Send + 'static {
let nonce = self.supervisor.snapshot_tracker.open();
let range = ..;
self.standard_tree()
self.tree
.create_range::<&[u8], _>(&range, nonce.instant, None)
.map(move |item| {
let _keep_snapshot_alive = &nonce;
@@ -472,7 +441,7 @@ impl Keyspace {
range: R,
) -> impl DoubleEndedIterator<Item = crate::Result<crate::KvPair>> + Send + 'static {
let nonce = self.supervisor.snapshot_tracker.open();
self.standard_tree()
self.tree
.create_range(&range, nonce.instant, None)
.map(move |item| {
let _keep_snapshot_alive = &nonce;
@@ -512,7 +481,7 @@ impl Keyspace {
prefix: K,
) -> impl DoubleEndedIterator<Item = crate::Result<crate::KvPair>> + Send + 'static {
let nonce = self.supervisor.snapshot_tracker.open();
self.standard_tree()
self.tree
.create_prefix(prefix, nonce.instant, None)
.map(move |item| {
let _keep_snapshot_alive = &nonce;
@@ -677,7 +646,7 @@ impl Keyspace {
&self,
key: K,
) -> crate::Result<Option<lsm_tree::UserValue>> {
Ok(self.standard_tree().get(key, SeqNo::MAX)?)
Ok(self.tree.get(key, SeqNo::MAX)?)
}
/// Retrieves the size of an item from the keyspace.
@@ -753,12 +722,6 @@ impl Keyspace {
self.tree.last_key_value(SeqNo::MAX, None).map(Guard)
}
/// Returns `true` if the underlying LSM-tree is key-value-separated.
#[must_use]
pub fn is_kv_separated(&self) -> bool {
matches!(self.tree, crate::AnyTree::Blob(_))
}
// NOTE: Used in tests
#[doc(hidden)]
pub fn rotate_memtable_and_wait(&self) -> crate::Result<()> {
@@ -908,13 +871,6 @@ impl Keyspace {
self.tree.table_count()
}
/// Number of blob files in the LSM-tree.
#[doc(hidden)]
#[must_use]
pub fn blob_file_count(&self) -> usize {
self.tree.blob_file_count()
}
/// Performs major compaction, blocking the caller until it's done.
///
/// # Errors
crates/fjall/src/keyspace/options.rs
+4 -226
View File
@@ -8,13 +8,11 @@ use crate::{
FilterPolicyEntry, HashRatioPolicy, PartitioningPolicy, PinningPolicy,
RestartIntervalPolicy,
},
keyspace::{config::DecodeConfig, InternalKeyspaceId},
meta_keyspace::{encode_config_key, MetaKeyspace},
keyspace::{InternalKeyspaceId, config::DecodeConfig},
meta_keyspace::{MetaKeyspace, encode_config_key},
};
use byteorder::ReadBytesExt;
use lsm_tree::{
compaction::Factory as CompactionFilterFactory, CompressionType, KvPair, KvSeparationOptions,
};
use lsm_tree::{CompressionType, KvPair, compaction::Factory as CompactionFilterFactory};
use std::sync::Arc;
/// Options to configure a keyspace
@@ -72,12 +70,6 @@ pub struct CreateOptions {
pub(crate) manual_journal_persist: bool,
#[doc(hidden)]
pub compaction_strategy: Arc<dyn lsm_tree::compaction::CompactionStrategy + Send + Sync>,
#[doc(hidden)]
pub kv_separation_opts: Option<KvSeparationOptions>,
pub(crate) compaction_filter_factory: Option<Arc<dyn CompactionFilterFactory>>,
}
@@ -120,12 +112,6 @@ impl Default for CreateOptions {
index_block_compression_policy: CompressionPolicy::all(CompressionType::None),
compaction_strategy: Arc::new(
crate::compaction::Leveled::default()
),
kv_separation_opts: None,
compaction_filter_factory: None,
}
}
@@ -153,8 +139,6 @@ impl CreateOptions {
keyspace_id: InternalKeyspaceId,
meta_keyspace: &MetaKeyspace,
) -> crate::Result<Self> {
let blob = meta_keyspace.get_kv_for_config(keyspace_id, "blob")?;
let data_block_compression_policy = meta_keyspace
.get_kv_for_config(keyspace_id, "data_block_compression_policy")?
.expect("should exist");
@@ -221,117 +205,6 @@ impl CreateOptions {
.expect("should exist");
let filter_policy = FilterPolicy::decode(&filter_policy)?;
let blob_opts = blob.map(|_| {
use byteorder::LE;
use lsm_tree::coding::Decode;
let blob_age_cutoff = meta_keyspace
.get_kv_for_config(keyspace_id, "blob_age_cutoff")?
.expect("blob_age_cutoff should be defined");
let blob_age_cutoff = (&mut &blob_age_cutoff[..]).read_f32::<LE>()?;
let blob_compression = meta_keyspace
.get_kv_for_config(keyspace_id, "blob_compression")?
.expect("blob_compression should be defined");
let blob_compression = CompressionType::decode_from(&mut &blob_compression[..])?;
let file_target_size = meta_keyspace
.get_kv_for_config(keyspace_id, "blob_file_target_size")?
.expect("blob_file_target_size should be defined");
let file_target_size = (&mut &file_target_size[..]).read_u64::<LE>()?;
let separation_threshold = meta_keyspace
.get_kv_for_config(keyspace_id, "blob_separation_threshold")?
.expect("blob_separation_threshold should be defined");
let separation_threshold = (&mut &separation_threshold[..]).read_u32::<LE>()?;
let staleness_threshold = meta_keyspace
.get_kv_for_config(keyspace_id, "blob_staleness_threshold")?
.expect("blob_staleness_threshold should be defined");
let staleness_threshold = (&mut &staleness_threshold[..]).read_f32::<LE>()?;
Ok::<_, crate::Error>(
KvSeparationOptions::default()
.compression(blob_compression)
.file_target_size(file_target_size)
.separation_threshold(separation_threshold)
.staleness_threshold(staleness_threshold)
.age_cutoff(blob_age_cutoff),
)
});
let compaction_strategy_name = meta_keyspace
.get_kv_for_config(keyspace_id, "compaction_strategy")?
.expect("compaction_strategy should be defined");
let compaction_strategy_name = std::str::from_utf8(&compaction_strategy_name)
.expect("compaction_strategy should be UTF-8");
let compaction_strategy = match compaction_strategy_name {
lsm_tree::compaction::LEVELED_COMPACTION_NAME => {
use byteorder::LE;
let l0_threshold = meta_keyspace
.get_kv_for_config(keyspace_id, "leveled_l0_threshold")?
.expect("leveled_l0_threshold should be defined");
let l0_threshold = (&mut &l0_threshold[..]).read_u8()?;
let target_size = meta_keyspace
.get_kv_for_config(keyspace_id, "leveled_target_size")?
.expect("leveled_target_size should be defined");
let target_size = (&mut &target_size[..]).read_u64::<LE>()?;
let level_ratio_policy_bytes = meta_keyspace
.get_kv_for_config(keyspace_id, "leveled_level_ratio_policy")?
.expect("leveled_level_ratio_policy should be defined");
let level_ratio_policy_bytes = &mut &level_ratio_policy_bytes[..];
let level_ratio_policy_len = level_ratio_policy_bytes.read_u8()?;
let mut ratios = vec![];
for _ in 0..level_ratio_policy_len {
ratios.push(level_ratio_policy_bytes.read_f32::<LE>()?);
}
Arc::new(
crate::compaction::Leveled::default()
.with_l0_threshold(l0_threshold)
.with_table_target_size(target_size)
.with_level_ratio_policy(ratios),
) as Arc<dyn lsm_tree::compaction::CompactionStrategy + Send + Sync>
}
lsm_tree::compaction::FIFO_COMPACTION_NAME => {
use byteorder::LE;
let fifo_limit = meta_keyspace
.get_kv_for_config(keyspace_id, "fifo_limit")?
.expect("fifo_limit should be defined");
let fifo_limit = (&mut &fifo_limit[..]).read_u64::<LE>()?;
let has_ttl = meta_keyspace
.get_kv_for_config(keyspace_id, "fifo_ttl")?
.expect("fifo_ttl should be defined")
== [1];
let ttl_seconds = if has_ttl {
let fifo_ttl_seconds = meta_keyspace
.get_kv_for_config(keyspace_id, "fifo_ttl_seconds")?
.expect("fifo_ttl_seconds should be defined");
let fifo_ttl_seconds = (&mut &fifo_ttl_seconds[..]).read_u64::<LE>()?;
Some(fifo_ttl_seconds)
} else {
None
};
Arc::new(crate::compaction::Fifo::new(fifo_limit, ttl_seconds))
}
name => {
panic!("Invalid/unsupported compaction strategy: {name:?}");
}
};
let manual_journal_persist = meta_keyspace
.get_kv_for_config(keyspace_id, "manual_journal_persist")?
.expect("should exist")
@@ -368,10 +241,6 @@ impl CreateOptions {
max_memtable_size,
compaction_strategy,
kv_separation_opts: blob_opts.transpose()?,
compaction_filter_factory: None,
})
}
@@ -380,11 +249,7 @@ impl CreateOptions {
pub(crate) fn encode_kvs(&self, keyspace_id: InternalKeyspaceId) -> Vec<KvPair> {
use crate::keyspace::config::EncodeConfig;
let mut kvs = vec![
{
let key = encode_config_key(keyspace_id, "compaction_strategy");
(key, self.compaction_strategy.get_name().into())
},
let kvs = vec![
policy!(
keyspace_id,
"data_block_compression_policy",
@@ -466,68 +331,9 @@ impl CreateOptions {
},
];
match self.compaction_strategy.get_name() {
"LeveledCompaction" | "FifoCompaction" => {
kvs.extend(
self.compaction_strategy
.get_config()
.into_iter()
.map(|(k, v)| {
// TODO: this is a bit stupid right now because we depend on behaviour in lsm-tree
// we should probably make lsm-tree just return a String for the key
let k = std::str::from_utf8(&k)
.expect("compaction strategy should return UTF-8 key");
(encode_config_key(keyspace_id, k), v)
}),
);
}
name => {
panic!("Invalid/unsupported compaction strategy: {name:?}");
}
}
if let Some(blob_opts) = &self.kv_separation_opts {
kvs.extend([
{
let key = encode_config_key(keyspace_id, "blob");
(key, [1u8].into())
},
{
let key = encode_config_key(keyspace_id, "blob_age_cutoff");
(key, blob_opts.age_cutoff.to_le_bytes().into())
},
{
use lsm_tree::coding::Encode;
let key = encode_config_key(keyspace_id, "blob_compression");
(key, blob_opts.compression.encode_into_vec().into())
},
{
let key = encode_config_key(keyspace_id, "blob_file_target_size");
(key, blob_opts.file_target_size.to_le_bytes().into())
},
{
let key = encode_config_key(keyspace_id, "blob_separation_threshold");
(key, blob_opts.separation_threshold.to_le_bytes().into())
},
{
let key = encode_config_key(keyspace_id, "blob_staleness_threshold");
(key, blob_opts.staleness_threshold.to_le_bytes().into())
},
]);
}
kvs
}
/// Toggles key-value separation.
#[must_use]
pub fn with_kv_separation(mut self, opts: Option<KvSeparationOptions>) -> Self {
self.kv_separation_opts = opts;
self
}
/// Sets the restart interval inside data blocks.
///
/// A higher restart interval saves space while increasing lookup times
@@ -632,18 +438,6 @@ impl CreateOptions {
self
}
/// Sets the compaction strategy.
///
/// Default = Leveled
#[must_use]
pub fn compaction_strategy(
mut self,
compaction_strategy: Arc<dyn lsm_tree::compaction::CompactionStrategy + Send + Sync>,
) -> Self {
self.compaction_strategy = compaction_strategy;
self
}
/// If `false`, writes will flush data to the operating system.
///
/// Default = false
@@ -704,23 +498,12 @@ mod tests {
c.data_block_compression_policy,
CompressionPolicy::disabled(),
);
assert_eq!(c.kv_separation_opts, None);
c = c.with_kv_separation(KvSeparationOptions::default());
assert_eq!(
c.kv_separation_opts.as_ref().unwrap().compression,
CompressionType::None,
);
c = c.data_block_compression_policy(CompressionPolicy::disabled());
assert_eq!(
c.data_block_compression_policy,
CompressionPolicy::disabled(),
);
assert_eq!(
c.kv_separation_opts.unwrap().compression,
CompressionType::None,
);
}
#[test]
@@ -734,16 +517,11 @@ mod tests {
c.data_block_compression_policy,
CompressionPolicy::new([Uncompressed, Uncompressed, Lz4]),
);
assert_eq!(c.kv_separation_opts, None);
c = c.with_kv_separation(Some(KvSeparationOptions::default()));
assert_eq!(c.kv_separation_opts.as_ref().unwrap().compression, Lz4);
c = c.data_block_compression_policy(CompressionPolicy::disabled());
assert_eq!(
c.data_block_compression_policy,
CompressionPolicy::disabled(),
);
assert_eq!(c.kv_separation_opts.as_ref().unwrap().compression, Lz4);
}
}
crates/fjall/src/lib.rs
+3 -20
View File
@@ -11,7 +11,6 @@
//! - Keyspaces (a.k.a. column families) with cross-keyspace atomic semantics
//! - Built-in compression (default = `LZ4`)
//! - Serializable transactions (optional)
//! - Key-value separation for large blob use cases (optional)
//! - Automatic background maintenance
//!
//! It is not:
@@ -98,10 +97,6 @@ pub mod compaction;
mod db_config;
#[cfg(feature = "__internal_whitebox")]
#[doc(hidden)]
pub mod drop;
mod db;
#[cfg(test)]
@@ -126,7 +121,6 @@ mod snapshot_nonce;
mod snapshot_tracker;
mod stats;
mod supervisor;
mod tx;
mod version;
mod worker_pool;
mod write_buffer_manager;
@@ -152,27 +146,16 @@ pub use {
guard::Guard,
iter::Iter,
journal::{error::RecoveryError as JournalRecoveryError, writer::PersistMode},
keyspace::{options::CreateOptions as KeyspaceCreateOptions, Keyspace},
keyspace::{Keyspace, options::CreateOptions as KeyspaceCreateOptions},
readable::Readable,
snapshot::Snapshot,
version::FormatVersion,
};
pub use tx::single_writer::{
SingleWriterTxKeyspace, TxDatabase as SingleWriterTxDatabase,
WriteTransaction as SingleWriterWriteTx,
};
pub use tx::optimistic::{
Conflict, OptimisticTxDatabase, OptimisticTxKeyspace, WriteTransaction as OptimisticWriteTx,
};
#[doc(hidden)]
pub use lsm_tree::{AbstractTree, AnyTree, Error as LsmError, TreeType};
pub use lsm_tree::{AbstractTree, Error as LsmError};
pub use lsm_tree::{
CompressionType, KvPair, KvSeparationOptions, SeqNo, Slice, UserKey, UserValue,
};
pub use lsm_tree::{CompressionType, KvPair, SeqNo, Slice, UserKey, UserValue};
/// Utility functions
pub mod util {
crates/fjall/src/locked_file.rs
+3 -1
View File
@@ -61,7 +61,9 @@ impl LockedFileGuard {
if let Err(e) = file.try_lock() {
match e {
std::fs::TryLockError::Error(e) => {
log::error!("Failed to acquire database lock - if this is expected, you can try opening again (maybe wait a little)");
log::error!(
"Failed to acquire database lock - if this is expected, you can try opening again (maybe wait a little)"
);
return Err(crate::Error::Io(e));
}
std::fs::TryLockError::WouldBlock => {
crates/fjall/src/meta_keyspace.rs
+10 -86
View File
@@ -2,13 +2,13 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{db::Keyspaces, keyspace::InternalKeyspaceId, Keyspace};
use crate::{Keyspace, db::Keyspaces, keyspace::InternalKeyspaceId};
use byteview::StrView;
use lsm_tree::{AbstractTree, AnyTree, SeqNo, SequenceNumberCounter, UserValue};
use lsm_tree::{AbstractTree, SeqNo, SequenceNumberCounter, Tree, UserValue};
use std::sync::{Arc, RwLock, RwLockWriteGuard};
pub fn encode_config_key(keyspace_id: InternalKeyspaceId, name: &str) -> crate::UserKey {
use byteorder::{WriteBytesExt, BE};
use byteorder::{BE, WriteBytesExt};
use std::io::Write;
#[expect(unsafe_code)]
@@ -46,7 +46,7 @@ pub struct MetaKeyspace {
///
/// We never commit data through the WAL but write tables directly using
/// lsm-tree's ingestion API.
inner: AnyTree,
inner: Tree,
/// Dictionary of all keyspaces
#[doc(hidden)]
@@ -58,7 +58,7 @@ pub struct MetaKeyspace {
impl MetaKeyspace {
pub(crate) fn new(
inner: AnyTree,
inner: Tree,
keyspaces: Arc<RwLock<Keyspaces>>,
seqno_generator: SequenceNumberCounter,
visible_seqno: SequenceNumberCounter,
@@ -224,7 +224,7 @@ impl MetaKeyspace {
#[cfg(test)]
mod tests {
use crate::{Database, Guard, KeyspaceCreateOptions, Readable, SingleWriterTxDatabase};
use crate::{Database, Guard, KeyspaceCreateOptions};
use test_log::test;
const ITEM_COUNT: usize = 10;
@@ -374,83 +374,6 @@ mod tests {
Ok(())
}
#[test]
fn tx_keyspace_delete() -> crate::Result<()> {
let folder = tempfile::tempdir()?;
let path;
{
let db = SingleWriterTxDatabase::builder(&folder).open()?;
let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
path = tree.path();
assert!(path.try_exists()?);
for x in 0..ITEM_COUNT as u64 {
let key = x.to_be_bytes();
let value = nanoid::nanoid!();
tree.insert(key, value.as_bytes())?;
}
for x in 0..ITEM_COUNT as u64 {
let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes();
let value = nanoid::nanoid!();
tree.insert(key, value.as_bytes())?;
}
assert_eq!(db.read_tx().len(&tree)?, ITEM_COUNT * 2);
assert_eq!(
db.read_tx().iter(&tree).flat_map(Guard::key).count(),
ITEM_COUNT * 2,
);
assert_eq!(
db.read_tx().iter(&tree).rev().flat_map(Guard::key).count(),
ITEM_COUNT * 2,
);
}
for _ in 0..3 {
let db = SingleWriterTxDatabase::builder(&folder).open()?;
let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
assert_eq!(db.read_tx().len(&tree)?, ITEM_COUNT * 2);
assert_eq!(
db.read_tx().iter(&tree).flat_map(Guard::key).count(),
ITEM_COUNT * 2,
);
assert_eq!(
db.read_tx().iter(&tree).rev().flat_map(Guard::key).count(),
ITEM_COUNT * 2,
);
assert!(path.try_exists()?);
}
{
let db = SingleWriterTxDatabase::builder(&folder).open()?;
{
let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
assert!(path.try_exists()?);
db.inner().delete_keyspace(tree.inner().clone())?;
}
assert!(!path.try_exists()?);
}
{
let _db = Database::builder(&folder).open()?;
assert!(!path.try_exists()?);
}
Ok(())
}
#[test]
fn keyspace_delete_and_reopening_behavior() -> crate::Result<()> {
let keyspace_name = "default";
@@ -473,9 +396,10 @@ mod tests {
db.delete_keyspace(keyspace)?;
assert!(!keyspace_exists(1)?);
assert!(db
.keyspace("default", KeyspaceCreateOptions::default)
.is_ok());
assert!(
db.keyspace("default", KeyspaceCreateOptions::default)
.is_ok()
);
assert!(keyspace_exists(2)?);
Ok(())
crates/fjall/src/poison.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use std::sync::{atomic::AtomicBool, Arc};
use std::sync::{Arc, atomic::AtomicBool};
#[derive(Clone, Debug, Default)]
pub struct PoisonSignal(Arc<AtomicBool>);
crates/fjall/src/recovery.rs
+2 -5
View File
@@ -3,15 +3,15 @@
// (found in the LICENSE-* files in the repository)
use crate::{
Database, HashMap, Keyspace,
file::{KEYSPACES_FOLDER, LSM_CURRENT_VERSION_MARKER},
journal::{
batch_reader::JournalBatchReader, manager::EvictionWatermark, reader::JournalReader,
},
keyspace::{
apply_to_base_config, options::CreateOptions as KeyspaceCreateOptions, InternalKeyspaceId,
InternalKeyspaceId, apply_to_base_config, options::CreateOptions as KeyspaceCreateOptions,
},
meta_keyspace::MetaKeyspace,
Database, HashMap, Keyspace,
};
use lsm_tree::AbstractTree;
use std::path::PathBuf;
@@ -177,9 +177,6 @@ pub fn recover_sealed_memtables(
lsm_tree::ValueType::WeakTombstone => {
tree.remove_weak(item.key, batch.seqno);
}
lsm_tree::ValueType::Indirection => {
unreachable!()
}
}
}
crates/fjall/src/snapshot.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{snapshot_nonce::SnapshotNonce, Guard, Iter, Keyspace, Readable};
use crate::{Guard, Iter, Keyspace, Readable, snapshot_nonce::SnapshotNonce};
use lsm_tree::{AbstractTree, SeqNo, UserValue};
use std::ops::RangeBounds;
crates/fjall/src/snapshot_nonce.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{snapshot_tracker::SnapshotTracker, SeqNo};
use crate::{SeqNo, snapshot_tracker::SnapshotTracker};
/// Holds a snapshot instant and automatically frees it from the snapshot tracker when dropped
pub struct SnapshotNonce {
crates/fjall/src/snapshot_tracker.rs
+2 -2
View File
@@ -2,10 +2,10 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{snapshot_nonce::SnapshotNonce, SeqNo};
use crate::{SeqNo, snapshot_nonce::SnapshotNonce};
use dashmap::DashMap;
use lsm_tree::SequenceNumberCounter;
use std::sync::{atomic::AtomicU64, Arc, RwLock};
use std::sync::{Arc, RwLock, atomic::AtomicU64};
/// Keeps track of open snapshots
pub struct SnapshotTrackerInner {
crates/fjall/src/supervisor.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use lsm_tree::SequenceNumberCounter;
use crate::{
db::Keyspaces,
flush::manager::FlushManager,
journal::{manager::JournalManager, Journal},
journal::{Journal, manager::JournalManager},
snapshot_tracker::SnapshotTracker,
write_buffer_manager::WriteBufferManager,
};
crates/fjall/src/tx/mod.rs
-7
View File
@@ -1,7 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
pub mod optimistic;
pub mod single_writer;
pub mod write_tx;
crates/fjall/src/tx/optimistic/conflict_manager.rs
-197
View File
@@ -1,197 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::keyspace::InternalKeyspaceId;
use core::ops::Bound;
use lsm_tree::Slice;
use std::{
collections::{BTreeMap, BTreeSet},
ops::RangeBounds,
sync::Mutex,
};
#[derive(Clone, Debug)]
enum Read {
Single(Slice),
Range {
start: Bound<Slice>,
end: Bound<Slice>,
},
All,
}
#[derive(Default, Debug)]
pub struct ConflictManager {
reads: Mutex<BTreeMap<InternalKeyspaceId, Vec<Read>>>,
conflict_keys: Mutex<BTreeMap<InternalKeyspaceId, BTreeSet<Slice>>>,
}
impl ConflictManager {
fn push_read(&self, keyspace_id: InternalKeyspaceId, read: Read) {
#[expect(clippy::expect_used)]
let mut lock = self.reads.lock().expect("lock is poisoned");
if let Some(tbl) = lock.get_mut(&keyspace_id) {
tbl.push(read);
} else {
lock.entry(keyspace_id).or_default().push(read);
}
}
pub fn mark_read(&self, keyspace_id: InternalKeyspaceId, key: Slice) {
self.push_read(keyspace_id, Read::Single(key));
}
pub fn mark_conflict(&self, keyspace_id: InternalKeyspaceId, key: Slice) {
#[expect(clippy::expect_used)]
let mut lock = self.conflict_keys.lock().expect("lock is poisoned");
if let Some(tbl) = lock.get_mut(&keyspace_id) {
tbl.insert(key);
} else {
lock.entry(keyspace_id).or_default().insert(key);
}
}
pub fn mark_range(&self, keyspace_id: InternalKeyspaceId, range: impl RangeBounds<Slice>) {
let start = match range.start_bound() {
Bound::Included(k) => Bound::Included(k.clone()),
Bound::Excluded(k) => Bound::Excluded(k.clone()),
Bound::Unbounded => Bound::Unbounded,
};
let end = match range.end_bound() {
Bound::Included(k) => Bound::Included(k.clone()),
Bound::Excluded(k) => Bound::Excluded(k.clone()),
Bound::Unbounded => Bound::Unbounded,
};
let read = if start == Bound::Unbounded && end == Bound::Unbounded {
Read::All
} else {
Read::Range { start, end }
};
self.push_read(keyspace_id, read);
}
#[expect(clippy::too_many_lines, clippy::significant_drop_tightening)]
pub fn has_conflict(&self, other: &Self) -> bool {
#[expect(clippy::expect_used)]
let reads_lock = self.reads.lock().expect("lock is poisoned");
#[expect(clippy::expect_used)]
let conflict_keys_lock = other.conflict_keys.lock().expect("lock is poisoned");
if reads_lock.is_empty() {
return false;
}
for (keyspace_name, keys) in &*reads_lock {
if let Some(other_conflict_keys) = conflict_keys_lock.get(keyspace_name) {
for ro in keys {
match ro {
Read::Single(k) => {
if other_conflict_keys.contains(k) {
return true;
}
}
Read::Range { start, end } => match (start, end) {
(Bound::Included(start), Bound::Included(end)) => {
if other_conflict_keys
.range::<Slice, _>((
Bound::Included(start),
Bound::Included(end),
))
.next()
.is_some()
{
return true;
}
}
(Bound::Included(start), Bound::Excluded(end)) => {
if other_conflict_keys
.range::<Slice, _>((
Bound::Included(start),
Bound::Excluded(end),
))
.next()
.is_some()
{
return true;
}
}
(Bound::Included(start), Bound::Unbounded) => {
if other_conflict_keys
.range::<Slice, _>((Bound::Included(start), Bound::Unbounded))
.next()
.is_some()
{
return true;
}
}
(Bound::Excluded(start), Bound::Included(end)) => {
if other_conflict_keys
.range::<Slice, _>((
Bound::Excluded(start),
Bound::Included(end),
))
.next()
.is_some()
{
return true;
}
}
(Bound::Excluded(start), Bound::Excluded(end)) => {
if other_conflict_keys
.range::<Slice, _>((
Bound::Excluded(start),
Bound::Excluded(end),
))
.next()
.is_some()
{
return true;
}
}
(Bound::Excluded(start), Bound::Unbounded) => {
if other_conflict_keys
.range::<Slice, _>((Bound::Excluded(start), Bound::Unbounded))
.next()
.is_some()
{
return true;
}
}
(Bound::Unbounded, Bound::Included(end)) => {
let range = ..=end;
for write in other_conflict_keys {
if range.contains(&write) {
return true;
}
}
}
(Bound::Unbounded, Bound::Excluded(end)) => {
let range = ..end;
for write in other_conflict_keys {
if range.contains(&write) {
return true;
}
}
}
(Bound::Unbounded, Bound::Unbounded) => unreachable!(),
},
Read::All => {
if !other_conflict_keys.is_empty() {
return true;
}
}
}
}
}
}
false
}
}
crates/fjall/src/tx/optimistic/keyspace.rs
-502
View File
@@ -1,502 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{tx::optimistic::OptimisticTxDatabase, Keyspace};
use crate::{Guard, Readable};
use lsm_tree::{UserKey, UserValue};
use std::path::PathBuf;
/// Handle to a keyspace of a transactional database
#[derive(Clone)]
pub struct OptimisticTxKeyspace {
pub(crate) inner: Keyspace,
pub(crate) db: OptimisticTxDatabase,
}
impl AsRef<Keyspace> for OptimisticTxKeyspace {
fn as_ref(&self) -> &Keyspace {
self.inner()
}
}
impl OptimisticTxKeyspace {
/// Returns the underlying LSM-tree's path.
#[must_use]
pub fn path(&self) -> PathBuf {
self.inner.path().into()
}
/// Approximates the number of items in the keyspace.
///
/// For update- or delete-heavy workloads, this value will
/// diverge from the real value, but is a O(1) operation.
///
/// For insert-only workloads (e.g. logs, time series)
/// this value is reliable.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// assert_eq!(tree.approximate_len(), 0);
///
/// tree.insert("1", "abc")?;
/// assert_eq!(tree.approximate_len(), 1);
///
/// tree.remove("1")?;
/// // Oops! approximate_len will not be reliable here
/// assert_eq!(tree.approximate_len(), 2);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[must_use]
pub fn approximate_len(&self) -> usize {
self.inner.approximate_len()
}
/// Removes an item and returns its value if it existed.
///
/// The operation will run wrapped in a transaction.
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let taken = tree.take("a")?.unwrap();
/// assert_eq!(b"abc", &*taken);
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn take<K: Into<UserKey>>(&self, key: K) -> crate::Result<Option<UserValue>> {
self.fetch_update(key, |_| None)
}
/// Atomically updates an item and returns the previous value.
///
/// Returning `None` removes the item if it existed before.
///
/// The operation will run wrapped in a transaction.
///
/// # Note
///
/// The provided closure can be called multiple times as this function
/// automatically retries on conflict. Since this is an `FnMut`, make sure
/// it is idempotent and will not cause side-effects.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, Slice, KeyspaceCreateOptions};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let prev = tree.fetch_update("a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"abc", &*prev);
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let prev = tree.fetch_update("a", |_| None)?.unwrap();
/// assert_eq!(b"abc", &*prev);
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn fetch_update<K: Into<UserKey>, F: FnMut(Option<&UserValue>) -> Option<UserValue>>(
&self,
key: K,
mut f: F,
) -> crate::Result<Option<UserValue>> {
let key = key.into();
loop {
let mut tx = self.db.write_tx()?;
let prev = tx.fetch_update(self.inner(), key.clone(), &mut f)?;
if tx.commit()?.is_ok() {
return Ok(prev);
}
}
}
/// Atomically updates an item and returns the new value.
///
/// Returning `None` removes the item if it existed before.
///
/// The operation will run wrapped in a transaction.
///
/// # Note
///
/// The provided closure can be called multiple times as this function
/// automatically retries on conflict. Since this is an `FnMut`, make sure
/// it is idempotent and will not cause side-effects.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, Slice, KeyspaceCreateOptions};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let updated = tree.update_fetch("a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"def", &*updated);
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let updated = tree.update_fetch("a", |_| None)?;
/// assert!(updated.is_none());
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn update_fetch<K: Into<UserKey>, F: FnMut(Option<&UserValue>) -> Option<UserValue>>(
&self,
key: K,
mut f: F,
) -> crate::Result<Option<UserValue>> {
let key = key.into();
loop {
let mut tx = self.db.write_tx()?;
let updated = tx.update_fetch(self.inner(), key.clone(), &mut f)?;
if tx.commit()?.is_ok() {
return Ok(updated);
}
}
}
/// Inserts a key-value pair into the keyspace.
///
/// Keys may be up to 65536 bytes long, values up to 2^32 bytes.
/// Shorter keys and values result in better performance.
///
/// If the key already exists, the item will be overwritten.
///
/// The operation will run wrapped in a transaction.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// assert!(!db.read_tx().is_empty(&tree)?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn insert<K: Into<UserKey>, V: Into<UserValue>>(
&self,
key: K,
value: V,
) -> crate::Result<()> {
let mut tx = self.db.write_tx()?;
tx.insert(self.inner(), key, value);
#[expect(
clippy::expect_used,
clippy::missing_panics_doc,
reason = "blind insert should not conflict ever"
)]
tx.commit()?.expect("blind insert should not conflict ever");
Ok(())
}
/// Removes an item from the keyspace.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
///
/// The operation will run wrapped in a transaction.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
/// assert!(!db.read_tx().is_empty(&tree)?);
///
/// tree.remove("a")?;
/// assert!(db.read_tx().is_empty(&tree)?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn remove<K: Into<UserKey>>(&self, key: K) -> crate::Result<()> {
let mut tx = self.db.write_tx()?;
tx.remove(self.inner(), key);
#[expect(
clippy::expect_used,
clippy::missing_panics_doc,
reason = "blind remove should not conflict ever"
)]
tx.commit()?.expect("blind remove should not conflict ever");
Ok(())
}
/// Removes an item from the keyspace, leaving behind a weak tombstone.
///
/// The tombstone marker of this delete operation will vanish when it
/// collides with its corresponding insertion.
/// This may cause older versions of the value to be resurrected, so it should
/// only be used and preferred in scenarios where a key is only ever written once.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
///
/// The operation will run wrapped in a transaction.
///
/// # Experimental
///
/// This function is currently experimental.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
/// assert!(!db.read_tx().is_empty(&tree)?);
///
/// tree.remove_weak("a")?;
/// assert!(db.read_tx().is_empty(&tree)?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
#[doc(hidden)]
pub fn remove_weak<K: Into<UserKey>>(&self, key: K) -> crate::Result<()> {
let mut tx = self.db.write_tx()?;
tx.remove_weak(self.inner(), key);
#[expect(clippy::expect_used)]
tx.commit()?.expect("blind remove should not conflict ever");
Ok(())
}
/// Retrieves an item from the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("my_value".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn get<K: AsRef<[u8]>>(&self, key: K) -> crate::Result<Option<lsm_tree::UserValue>> {
self.inner.get(key)
}
/// Retrieves the size of an item from the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
///
/// let len = tree.size_of("a")?.unwrap_or_default();
/// assert_eq!("my_value".len() as u32, len);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn size_of<K: AsRef<[u8]>>(&self, key: K) -> crate::Result<Option<u32>> {
self.inner.size_of(key)
}
/// Returns the first key-value pair in the keyspace.
/// The key in this pair is the minimum key in the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
/// tree.insert("b", "my_value")?;
///
/// assert_eq!(b"a", &*tree.first_key_value().unwrap().key()?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[must_use]
pub fn first_key_value(&self) -> Option<Guard> {
let read_tx = self.db.read_tx();
read_tx.first_key_value(self)
}
/// Returns the last key-value pair in the keyspace.
/// The key in this pair is the maximum key in the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
/// tree.insert("b", "my_value")?;
///
/// assert_eq!(b"b", &*tree.last_key_value().unwrap().key()?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[must_use]
pub fn last_key_value(&self) -> Option<Guard> {
let read_tx = self.db.read_tx();
read_tx.last_key_value(self)
}
/// Returns `true` if the keyspace contains the specified key.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
///
/// assert!(tree.contains_key("a")?);
/// assert!(!tree.contains_key("b")?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn contains_key<K: AsRef<[u8]>>(&self, key: K) -> crate::Result<bool> {
self.inner.contains_key(key)
}
/// Allows access to the inner keyspace handle, allowing to
/// escape from the transactional context.
#[doc(hidden)]
#[must_use]
pub fn inner(&self) -> &Keyspace {
&self.inner
}
}
crates/fjall/src/tx/optimistic/mod.rs
-184
View File
@@ -1,184 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
mod conflict_manager;
mod keyspace;
mod oracle;
mod write_tx;
use crate::{
keyspace::KeyspaceKey,
tx::{optimistic::oracle::Oracle, single_writer::Openable},
Config, Database, KeyspaceCreateOptions, PersistMode, Snapshot,
};
use std::{
path::Path,
sync::{Arc, Mutex},
};
pub use keyspace::OptimisticTxKeyspace;
pub use write_tx::{Conflict, WriteTransaction};
/// Transactional database
#[derive(Clone)]
pub struct OptimisticTxDatabase {
pub(crate) inner: Database,
pub(super) oracle: Arc<Oracle>,
}
impl Openable for OptimisticTxDatabase {
fn open(config: Config) -> crate::Result<Self>
where
Self: Sized,
{
let inner = Database::create_or_recover(config)?;
// inner.start_background_threads()?;
Ok(Self {
oracle: Arc::new(Oracle {
write_serialize_lock: Mutex::default(),
snapshot_tracker: inner.supervisor.snapshot_tracker.clone(),
}),
inner,
})
}
}
impl OptimisticTxDatabase {
/// Creates a new database builder to create or open a database at `path`.
pub fn builder(path: impl AsRef<Path>) -> crate::DatabaseBuilder<Self> {
crate::DatabaseBuilder::new(path.as_ref())
}
#[doc(hidden)]
#[must_use]
pub fn inner(&self) -> &Database {
&self.inner
}
/// Starts a new writeable transaction.
///
/// # Errors
///
/// Will return `Err` if creation failed.
pub fn write_tx(&self) -> crate::Result<WriteTransaction> {
let snapshot = {
// acquire a lock here to prevent getting a stale snapshot seqno
// this will drain at least part of the commit queue, but ordering
// is platform-dependent since we use std::sync::Mutex
let _guard = self.oracle.write_serialize_lock()?;
self.inner.supervisor.snapshot_tracker.open()
};
let mut write_tx =
WriteTransaction::new(self.inner().clone(), snapshot, self.oracle.clone());
if !self.inner.config.manual_journal_persist {
write_tx = write_tx.durability(Some(PersistMode::Buffer));
}
Ok(write_tx)
}
/// Starts a new read-only transaction (a.k.a. [`Snapshot`]).
#[must_use]
pub fn read_tx(&self) -> Snapshot {
self.inner.snapshot()
}
/// Flushes the active journal. The durability depends on the [`PersistMode`]
/// used.
///
/// Persisting only affects durability, NOT consistency! Even without flushing
/// data is crash-safe.
///
/// # Examples
///
/// ```
/// # use fjall::{PersistMode, OptimisticTxDatabase, KeyspaceCreateOptions};
/// # let folder = tempfile::tempdir()?;
/// let db = OptimisticTxDatabase::builder(folder).open()?;
/// let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?;
///
/// items.insert("a", "hello")?;
///
/// db.persist(PersistMode::SyncAll)?;
/// #
/// # Ok::<_, fjall::Error>(())
/// ```
///
/// # Errors
///
/// Returns error, if an IO error occurred.
pub fn persist(&self, mode: PersistMode) -> crate::Result<()> {
self.inner.persist(mode)
}
/// Creates or opens a keyspace.
///
/// If the keyspace does not yet exist, it will be created configured with `create_options`.
/// Otherwise simply a handle to the existing keyspace will be returned.
///
/// Keyspace names can be up to 255 characters long and can not be empty.
///
/// # Errors
///
/// Returns error, if an IO error occurred.
///
/// # Panics
///
/// Panics if the keyspace name is invalid.
pub fn keyspace(
&self,
name: &str,
create_options: impl FnOnce() -> KeyspaceCreateOptions,
) -> crate::Result<OptimisticTxKeyspace> {
let keyspace = self.inner.keyspace(name, create_options)?;
Ok(OptimisticTxKeyspace {
inner: keyspace,
db: self.clone(),
})
}
/// Returns the number of keyspaces.
#[must_use]
pub fn keyspace_count(&self) -> usize {
self.inner.keyspace_count()
}
/// Gets a list of all keyspace names in the database.
#[must_use]
pub fn list_keyspace_names(&self) -> Vec<KeyspaceKey> {
self.inner.list_keyspace_names()
}
/// Returns `true` if the keyspace with the given name exists.
#[must_use]
pub fn keyspace_exists(&self, name: &str) -> bool {
self.inner.keyspace_exists(name)
}
/// Returns the current write buffer size (active + sealed memtables).
#[must_use]
pub fn write_buffer_size(&self) -> u64 {
self.inner.write_buffer_size()
}
/// Returns the number of journal fragments on disk.
#[must_use]
pub fn journal_count(&self) -> usize {
self.inner.journal_count()
}
/// Returns the disk space usage of the entire database.
///
/// # Errors
///
/// Returns error, if an IO error occurred.
pub fn disk_space(&self) -> crate::Result<u64> {
self.inner.disk_space()
}
}
crates/fjall/src/tx/optimistic/oracle.rs
-121
View File
@@ -1,121 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use super::conflict_manager::ConflictManager;
use crate::snapshot_tracker::SnapshotTracker;
use crate::SeqNo;
use std::collections::BTreeMap;
use std::sync::{Mutex, MutexGuard};
pub enum CommitOutcome<E> {
Ok,
Aborted(E),
Conflicted,
}
pub struct Oracle {
pub(super) write_serialize_lock: Mutex<BTreeMap<u64, ConflictManager>>,
pub(super) snapshot_tracker: SnapshotTracker,
}
impl Oracle {
pub(super) fn with_commit<E, F: FnOnce() -> Result<(), E>>(
&self,
instant: SeqNo,
conflict_checker: ConflictManager,
f: F,
) -> crate::Result<CommitOutcome<E>> {
let mut committed_txns = self
.write_serialize_lock
.lock()
.map_err(|_| crate::Error::Poisoned)?;
// If the committed_txn.ts is less than `SeqNo` that implies that the
// committed_txn finished before the current transaction started.
// We don't need to check for conflict in that case.
// This change assumes linearizability. Lack of linearizability could
// cause the read ts of a new txn to be lower than the commit ts of
// a txn before it.
let conflicted =
committed_txns
.range((instant + 1)..)
.any(|(_ts, other_conflict_checker)| {
conflict_checker.has_conflict(other_conflict_checker)
});
self.snapshot_tracker.close_raw(instant);
// TODO: This can be expensive and should probably be done in a background worker, or a after a memtable rotation
let safe_to_gc = self.snapshot_tracker.get_seqno_safe_to_gc();
committed_txns.retain(|ts, _| *ts > safe_to_gc);
if conflicted {
return Ok(CommitOutcome::Conflicted);
}
if let Err(e) = f() {
return Ok(CommitOutcome::Aborted(e));
}
committed_txns.insert(self.snapshot_tracker.get(), conflict_checker);
Ok(CommitOutcome::Ok)
}
pub(super) fn write_serialize_lock(
&self,
) -> crate::Result<MutexGuard<'_, BTreeMap<u64, ConflictManager>>> {
self.write_serialize_lock
.lock()
.map_err(|_| crate::Error::Poisoned)
}
}
#[cfg(test)]
mod tests {
use crate::{KeyspaceCreateOptions, OptimisticTxDatabase, OptimisticTxKeyspace, Readable};
#[expect(clippy::significant_drop_tightening)]
fn run_tx(
db: &OptimisticTxDatabase,
tree: &OptimisticTxKeyspace,
) -> Result<(), Box<dyn std::error::Error>> {
let mut tx1 = db.write_tx()?;
let mut tx2 = db.write_tx()?;
tx1.insert(tree.inner(), "hello", "world");
tx1.commit()??;
assert!(tree.contains_key("hello")?);
_ = tx2.get(tree.inner(), "hello")?;
tx2.insert(tree.inner(), "hello", "world2");
assert!(tx2.commit()?.is_err()); // intended to conflict
Ok(())
}
#[expect(clippy::unwrap_used)]
#[test]
fn oracle_committed_txns_does_not_leak() -> crate::Result<()> {
let tmpdir = tempfile::tempdir()?;
let db = OptimisticTxDatabase::builder(tmpdir.path()).open()?;
let part = db.keyspace("foo", KeyspaceCreateOptions::default)?;
for _ in 0..10_000 {
run_tx(&db, &part).unwrap();
}
assert!(dbg!(db.oracle.write_serialize_lock.lock().unwrap().len()) < 10_000);
for _ in 0..10_000 {
run_tx(&db, &part).unwrap();
}
assert!(dbg!(db.oracle.write_serialize_lock.lock().unwrap().len()) < 10_000);
Ok(())
}
}
crates/fjall/src/tx/optimistic/write_tx.rs
-891
View File
@@ -1,891 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{
snapshot_nonce::SnapshotNonce,
tx::{
optimistic::{
conflict_manager::ConflictManager,
oracle::{CommitOutcome, Oracle},
},
write_tx::BaseTransaction,
},
Database, Guard, Iter, Keyspace, PersistMode, Readable,
};
use lsm_tree::{Slice, UserKey, UserValue};
use std::{
fmt,
ops::{Bound, RangeBounds, RangeFull},
sync::Arc,
};
/// Transaction conflict
///
/// SSI transactions can conflict which require them to be rerun.
#[derive(Debug)]
pub struct Conflict;
impl std::error::Error for Conflict {}
impl fmt::Display for Conflict {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
"Transaction conflict".fmt(f)
}
}
/// A cross-keyspace transaction using optimistic concurrency control
///
/// Use [`WriteTransaction::commit`] to commit changes to the keyspace(s).
///
/// Transactions keep a consistent view of the database at the time,
/// meaning old data will not be dropped until it is not referenced by any active transaction.
///
/// For that reason, you should try to keep transactions short-lived, and make sure they
/// are not held somewhere forever.
///
/// # Caution
///
/// The transaction may fail and have to be rerun if it conflicts.
#[clippy::has_significant_drop]
pub struct WriteTransaction {
inner: BaseTransaction,
cm: ConflictManager,
oracle: Arc<Oracle>,
}
impl Readable for WriteTransaction {
fn get<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<Option<UserValue>> {
let keyspace = keyspace.as_ref();
let res = self.inner.get(keyspace, key.as_ref())?;
self.cm.mark_read(keyspace.id, key.as_ref().into());
Ok(res)
}
fn contains_key<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<bool> {
let keyspace = keyspace.as_ref();
let contains = self.inner.contains_key(keyspace, key.as_ref())?;
self.cm.mark_read(keyspace.id, key.as_ref().into());
Ok(contains)
}
fn first_key_value(&self, keyspace: impl AsRef<Keyspace>) -> Option<Guard> {
self.iter(&keyspace).next()
}
fn last_key_value(&self, keyspace: impl AsRef<Keyspace>) -> Option<Guard> {
self.iter(&keyspace).next_back()
}
fn size_of<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<Option<u32>> {
let keyspace = keyspace.as_ref();
self.inner.size_of(keyspace, key)
}
fn iter(&self, keyspace: impl AsRef<Keyspace>) -> Iter {
self.cm.mark_range(keyspace.as_ref().id, RangeFull);
self.inner.iter(keyspace)
}
fn range<K: AsRef<[u8]>, R: RangeBounds<K>>(
&self,
keyspace: impl AsRef<Keyspace>,
range: R,
) -> Iter {
let start: Bound<Slice> = range.start_bound().map(|k| k.as_ref().into());
let end: Bound<Slice> = range.end_bound().map(|k| k.as_ref().into());
self.cm.mark_range(keyspace.as_ref().id, (start, end));
self.inner.range(keyspace, range)
}
fn prefix<K: AsRef<[u8]>>(&self, keyspace: impl AsRef<Keyspace>, prefix: K) -> Iter {
self.range(keyspace, lsm_tree::range::prefix_to_range(prefix.as_ref()))
}
}
impl WriteTransaction {
pub(crate) fn new(db: Database, nonce: SnapshotNonce, oracle: Arc<Oracle>) -> Self {
Self {
inner: BaseTransaction::new(db, nonce),
cm: ConflictManager::default(),
oracle,
}
}
/// Sets the durability level.
#[must_use]
pub fn durability(mut self, mode: Option<PersistMode>) -> Self {
self.inner = self.inner.durability(mode);
self
}
/// Removes an item and returns its value if it existed.
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx()?;
///
/// let taken = tx.take(&tree, "a")?.unwrap();
/// assert_eq!(b"abc", &*taken);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn take<K: Into<UserKey>>(
&mut self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<Option<UserValue>> {
self.fetch_update(keyspace, key, |_| None)
}
/// Atomically updates an item and returns the new value.
///
/// Returning `None` removes the item if it existed before.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable, Slice};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx()?;
///
/// let updated = tx.update_fetch(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"def", &*updated);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx()?;
///
/// let updated = tx.update_fetch(&tree, "a", |_| None)?;
/// assert!(updated.is_none());
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn update_fetch<K: Into<UserKey>, F: FnOnce(Option<&UserValue>) -> Option<UserValue>>(
&mut self,
keyspace: impl AsRef<Keyspace>,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
let keyspace = keyspace.as_ref();
let key: UserKey = key.into();
let updated = self.inner.update_fetch(keyspace, key.clone(), f)?;
self.cm.mark_read(keyspace.id, key.clone());
self.cm.mark_conflict(keyspace.id, key);
Ok(updated)
}
/// Atomically updates an item and returns the previous value.
///
/// Returning `None` removes the item if it existed before.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable, Slice};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx()?;
///
/// let prev = tx.fetch_update(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"abc", &*prev);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx()?;
///
/// let prev = tx.fetch_update(&tree, "a", |_| None)?.unwrap();
/// assert_eq!(b"abc", &*prev);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn fetch_update<K: Into<UserKey>, F: FnOnce(Option<&UserValue>) -> Option<UserValue>>(
&mut self,
keyspace: impl AsRef<Keyspace>,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
let keyspace = keyspace.as_ref();
let key = key.into();
let prev = self.inner.fetch_update(keyspace, key.clone(), f)?;
self.cm.mark_read(keyspace.id, key.clone());
self.cm.mark_conflict(keyspace.id, key);
Ok(prev)
}
/// Inserts a key-value pair into the keyspace.
///
/// Keys may be up to 65536 bytes long, values up to 2^32 bytes.
/// Shorter keys and values result in better performance.
///
/// If the key already exists, the item will be overwritten.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "previous_value")?;
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
///
/// let mut tx = db.write_tx()?;
/// tx.insert(&tree, "a", "new_value");
///
/// drop(tx);
///
/// // Write was not committed
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
pub fn insert<K: Into<UserKey>, V: Into<UserValue>>(
&mut self,
keyspace: impl AsRef<Keyspace>,
key: K,
value: V,
) {
let keyspace = keyspace.as_ref();
let key: UserKey = key.into();
self.inner.insert(keyspace, key.clone(), value);
self.cm.mark_conflict(keyspace.id, key);
}
/// Removes an item from the keyspace.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "previous_value")?;
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
///
/// let mut tx = db.write_tx()?;
/// tx.remove(&tree, "a");
///
/// // Read-your-own-write
/// let item = tx.get(&tree, "a")?;
/// assert_eq!(None, item);
///
/// drop(tx);
///
/// // Deletion was not committed
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
pub fn remove<K: Into<UserKey>>(&mut self, keyspace: impl AsRef<Keyspace>, key: K) {
let keyspace = keyspace.as_ref();
let key: UserKey = key.into();
self.inner.remove(keyspace, key.clone());
self.cm.mark_conflict(keyspace.id, key);
}
/// Removes an item from the keyspace, leaving behind a weak tombstone.
///
/// The tombstone marker of this delete operation will vanish when it
/// collides with its corresponding insertion.
/// This may cause older versions of the value to be resurrected, so it should
/// only be used and preferred in scenarios where a key is only ever written once.
///
/// # Experimental
///
/// This function is currently experimental.
///
/// # Examples
///
/// ```
/// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = OptimisticTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "previous_value")?;
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
///
/// let mut tx = db.write_tx()?;
/// tx.remove_weak(&tree, "a");
///
/// // Read-your-own-write
/// let item = tx.get(&tree, "a")?;
/// assert_eq!(None, item);
///
/// drop(tx);
///
/// // Deletion was not committed
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[doc(hidden)]
pub fn remove_weak<K: Into<UserKey>>(&mut self, keyspace: impl AsRef<Keyspace>, key: K) {
let keyspace = keyspace.as_ref();
let key: UserKey = key.into();
self.inner.remove_weak(keyspace, key.clone());
self.cm.mark_conflict(keyspace.id, key);
}
/// Commits the transaction.
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn commit(self) -> crate::Result<Result<(), Conflict>> {
// NOTE: We have no write set, so we are basically
// a read-only transaction, so nothing to do here
if self.inner.memtables.is_empty() {
return Ok(Ok(()));
}
let oracle = self.oracle.clone();
match oracle.with_commit(self.inner.nonce.instant, self.cm, move || {
self.inner.commit()
})? {
CommitOutcome::Ok => Ok(Ok(())),
CommitOutcome::Aborted(e) => Err(e),
CommitOutcome::Conflicted => Ok(Err(Conflict)),
}
}
/// More explicit alternative to dropping the transaction
/// to roll it back.
pub fn rollback(self) {
self.inner.rollback();
}
}
#[cfg(test)]
mod tests {
use crate::{
Conflict, KeyspaceCreateOptions, OptimisticTxDatabase, OptimisticTxKeyspace, Readable,
};
use tempfile::TempDir;
use test_log::test;
struct TestEnv {
db: OptimisticTxDatabase,
tree: OptimisticTxKeyspace,
#[expect(unused)]
tmpdir: TempDir,
}
impl TestEnv {
fn seed_hermitage_data(&self) -> crate::Result<()> {
self.tree.insert([1u8], [10u8])?;
self.tree.insert([2u8], [20u8])?;
Ok(())
}
}
fn setup() -> Result<TestEnv, Box<dyn std::error::Error>> {
let tmpdir = tempfile::tempdir()?;
let db = OptimisticTxDatabase::builder(tmpdir.path()).open()?;
let tree = db.keyspace("foo", KeyspaceCreateOptions::default)?;
Ok(TestEnv { db, tree, tmpdir })
}
// Adapted from https://github.com/al8n/skipdb/issues/10
#[test]
#[expect(clippy::unwrap_used)]
fn tx_ssi_arthur_1() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
let mut tx = env.db.write_tx()?;
tx.insert(env.tree.inner(), "a1", 10u64.to_be_bytes());
tx.insert(env.tree.inner(), "b1", 100u64.to_be_bytes());
tx.insert(env.tree.inner(), "b2", 200u64.to_be_bytes());
tx.commit()??;
let mut tx1 = env.db.write_tx()?;
let val = tx1
.range(&env.tree, "a".."b")
.map(|kv| {
let v = kv.value().unwrap();
let mut buf = [0u8; 8];
buf.copy_from_slice(&v);
u64::from_be_bytes(buf)
})
.sum::<u64>();
tx1.insert(env.tree.inner(), "b3", 10u64.to_be_bytes());
assert_eq!(10, val);
let mut tx2 = env.db.write_tx()?;
let val = tx2
.range(&env.tree, "b".."c")
.map(|kv| {
let v = kv.value().unwrap();
let mut buf = [0u8; 8];
buf.copy_from_slice(&v);
u64::from_be_bytes(buf)
})
.sum::<u64>();
tx2.insert(env.tree.inner(), "a3", 300u64.to_be_bytes());
assert_eq!(300, val);
tx2.commit()??;
assert!(matches!(tx1.commit()?, Err(Conflict)));
let tx3 = env.db.write_tx()?;
let val = tx3
.iter(&env.tree)
.filter_map(|kv| {
let (k, v) = kv.into_inner().unwrap();
if k.starts_with(b"a") {
let mut buf = [0u8; 8];
buf.copy_from_slice(&v);
Some(u64::from_be_bytes(buf))
} else {
None
}
})
.sum::<u64>();
assert_eq!(310, val);
Ok(())
}
// Adapted from https://github.com/al8n/skipdb/issues/10
#[test]
#[expect(clippy::unwrap_used)]
fn tx_ssi_arthur_2() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
let mut tx = env.db.write_tx()?;
tx.insert(env.tree.inner(), "b1", 100u64.to_be_bytes());
tx.insert(env.tree.inner(), "b2", 200u64.to_be_bytes());
tx.commit()??;
let mut tx1 = env.db.write_tx()?;
let val = tx1
.range(&env.tree, "a".."b")
.map(|kv| {
let v = kv.value().unwrap();
let mut buf = [0u8; 8];
buf.copy_from_slice(&v);
u64::from_be_bytes(buf)
})
.sum::<u64>();
tx1.insert(env.tree.inner(), "b3", 0u64.to_be_bytes());
assert_eq!(0, val);
let mut tx2 = env.db.write_tx()?;
let val = tx2
.range(&env.tree, "b".."c")
.map(|kv| {
let v = kv.value().unwrap();
let mut buf = [0u8; 8];
buf.copy_from_slice(&v);
u64::from_be_bytes(buf)
})
.sum::<u64>();
tx2.insert(env.tree.inner(), "a3", 300u64.to_be_bytes());
assert_eq!(300, val);
tx2.commit()??;
assert!(matches!(tx1.commit()?, Err(Conflict)));
let tx3 = env.db.write_tx()?;
let val = tx3
.iter(&env.tree)
.filter_map(|kv| {
let (k, v) = kv.into_inner().unwrap();
if k.starts_with(b"a") {
let mut buf = [0u8; 8];
buf.copy_from_slice(&v);
Some(u64::from_be_bytes(buf))
} else {
None
}
})
.sum::<u64>();
assert_eq!(300, val);
Ok(())
}
#[test]
fn tx_ssi_basic() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
let mut tx1 = env.db.write_tx()?;
let mut tx2 = env.db.write_tx()?;
tx1.insert(env.tree.inner(), "hello", "world");
tx1.commit()??;
assert!(env.tree.contains_key("hello")?);
assert_eq!(tx2.get(env.tree.inner(), "hello")?, None);
tx2.insert(env.tree.inner(), "hello", "world2");
assert!(matches!(tx2.commit()?, Err(Conflict)));
let mut tx1 = env.db.write_tx()?;
let mut tx2 = env.db.write_tx()?;
tx1.iter(&env.tree).next();
tx2.insert(env.tree.inner(), "hello", "world2");
tx1.insert(env.tree.inner(), "hello2", "world1");
tx1.commit()??;
tx2.commit()??;
Ok(())
}
#[test]
#[expect(clippy::unwrap_used)]
fn tx_ssi_ww() -> Result<(), Box<dyn std::error::Error>> {
// https://en.wikipedia.org/wiki/Write%E2%80%93write_conflict
let env = setup()?;
let mut tx1 = env.db.write_tx()?;
let mut tx2 = env.db.write_tx()?;
tx1.insert(env.tree.inner(), "a", "a");
tx2.insert(env.tree.inner(), "b", "c");
tx1.insert(env.tree.inner(), "b", "b");
tx1.commit()??;
tx2.insert(env.tree.inner(), "a", "c");
tx2.commit()??;
assert_eq!(b"c", &*env.tree.get("a")?.unwrap());
assert_eq!(b"c", &*env.tree.get("b")?.unwrap());
Ok(())
}
#[test]
#[expect(clippy::unwrap_used)]
fn tx_ssi_swap() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
env.tree.insert("x", "x")?;
env.tree.insert("y", "y")?;
let mut tx1 = env.db.write_tx()?;
let mut tx2 = env.db.write_tx()?;
{
let x = tx1.get(env.tree.inner(), "x")?.unwrap();
tx1.insert(env.tree.inner(), "y", x);
}
{
let y = tx2.get(env.tree.inner(), "y")?.unwrap();
tx2.insert(env.tree.inner(), "x", y);
}
tx1.commit()??;
assert!(matches!(tx2.commit()?, Err(Conflict)));
Ok(())
}
#[test]
fn tx_ssi_write_cycles() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
env.seed_hermitage_data()?;
let mut t1 = env.db.write_tx()?;
let mut t2 = env.db.write_tx()?;
t1.insert(env.tree.inner(), [1u8], [11u8]);
t2.insert(env.tree.inner(), [1u8], [12u8]);
t1.insert(env.tree.inner(), [2u8], [21u8]);
t1.commit()??;
assert_eq!(env.tree.get([1u8])?, Some([11u8].into()));
t2.insert(env.tree.inner(), [2u8], [22u8]);
t2.commit()??;
assert_eq!(env.tree.get([1u8])?, Some([12u8].into()));
assert_eq!(env.tree.get([2u8])?, Some([22u8].into()));
Ok(())
}
#[test]
fn tx_ssi_aborted_reads() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
env.seed_hermitage_data()?;
let mut t1 = env.db.write_tx()?;
let t2 = env.db.write_tx()?;
t1.insert(env.tree.inner(), [1u8], [101u8]);
assert_eq!(t2.get(env.tree.inner(), [1u8])?, Some([10u8].into()));
t1.rollback();
assert_eq!(t2.get(env.tree.inner(), [1u8])?, Some([10u8].into()));
t2.commit()??;
Ok(())
}
#[expect(clippy::unwrap_used)]
#[test]
fn tx_ssi_anti_dependency_cycles() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
env.seed_hermitage_data()?;
let mut t1 = env.db.write_tx()?;
{
let mut iter = t1.iter(&env.tree);
assert_eq!(
iter.next().unwrap().into_inner()?,
([1u8].into(), [10u8].into()),
);
assert_eq!(
iter.next().unwrap().into_inner()?,
([2u8].into(), [20u8].into()),
);
assert!(iter.next().is_none());
}
let mut t2 = env.db.write_tx()?;
let new = t2.update_fetch(&env.tree, [2u8], |v| {
v.and_then(|v| v.first().copied()).map(|v| [v + 5].into())
})?;
assert_eq!(new, Some([25u8].into()));
t2.commit()??;
let t3 = env.db.write_tx()?;
{
let mut iter = t3.iter(&env.tree);
assert_eq!(
iter.next().unwrap().into_inner()?,
([1u8].into(), [10u8].into()),
);
assert_eq!(
iter.next().unwrap().into_inner()?,
([2u8].into(), [25u8].into()),
); // changed here
assert!(iter.next().is_none());
}
t3.commit()??;
t1.insert(env.tree.inner(), [1u8], [0u8]);
assert!(matches!(t1.commit()?, Err(Conflict)));
Ok(())
}
#[test]
fn tx_ssi_update_fetch_update() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
let mut t1 = env.db.write_tx()?;
let mut t2 = env.db.write_tx()?;
let new = t1.update_fetch(env.tree.inner(), "hello", |_| Some("world".into()))?;
assert_eq!(new, Some("world".into()));
let old = t2.fetch_update(env.tree.inner(), "hello", |_| Some("world2".into()))?;
assert_eq!(old, None);
t1.commit()??;
assert!(matches!(t2.commit()?, Err(Conflict)));
assert_eq!(env.tree.get("hello")?, Some("world".into()));
let mut t1 = env.db.write_tx()?;
let mut t2 = env.db.write_tx()?;
let old = t1.fetch_update(env.tree.inner(), "hello", |_| Some("world3".into()))?;
assert_eq!(old, Some("world".into()));
let new = t2.update_fetch(env.tree.inner(), "hello2", |_| Some("world2".into()))?;
assert_eq!(new, Some("world2".into()));
t1.commit()??;
t2.commit()??;
assert_eq!(env.tree.get("hello")?, Some("world3".into()));
assert_eq!(env.tree.get("hello2")?, Some("world2".into()));
Ok(())
}
#[test]
fn tx_ssi_range() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
let mut t1 = env.db.write_tx()?;
let mut t2 = env.db.write_tx()?;
_ = t1.range(&env.tree, "h"..="hello");
t1.insert(env.tree.inner(), "foo", "bar");
// insert a key INSIDE the range read by t1
t2.insert(env.tree.inner(), "hello", "world");
t2.commit()??;
assert!(matches!(t1.commit()?, Err(Conflict)));
let mut t1 = env.db.write_tx()?;
let mut t2 = env.db.write_tx()?;
_ = t1.range(&env.tree, "h"..="hello");
t1.insert(env.tree.inner(), "foo", "bar");
// insert a key OUTSIDE the range read by t1
t2.insert(env.tree.inner(), "hello2", "world");
t2.commit()??;
t1.commit()??;
Ok(())
}
#[test]
fn tx_ssi_prefix() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
let mut t1 = env.db.write_tx()?;
let mut t2 = env.db.write_tx()?;
_ = t1.prefix(&env.tree, "hello");
t1.insert(env.tree.inner(), "foo", "bar");
// insert a key MATCHING the prefix read by t1
t2.insert(env.tree.inner(), "hello", "world");
t2.commit()??;
assert!(matches!(t1.commit()?, Err(Conflict)));
let mut t1 = env.db.write_tx()?;
let mut t2 = env.db.write_tx()?;
_ = t1.prefix(&env.tree, "hello");
t1.insert(env.tree.inner(), "foo", "bar");
// insert a key NOT MATCHING the range read by t1
t2.insert(env.tree.inner(), "foobar", "world");
t2.commit()??;
t1.commit()??;
Ok(())
}
}
crates/fjall/src/tx/single_writer/keyspace.rs
-469
View File
@@ -1,469 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{Guard, Keyspace, Readable, SingleWriterTxDatabase};
use lsm_tree::{UserKey, UserValue};
use std::path::PathBuf;
/// Handle to a keyspace of a transactional database
#[derive(Clone)]
pub struct SingleWriterTxKeyspace {
pub(crate) inner: Keyspace,
pub(crate) db: SingleWriterTxDatabase,
}
impl AsRef<Keyspace> for SingleWriterTxKeyspace {
fn as_ref(&self) -> &Keyspace {
self.inner()
}
}
impl SingleWriterTxKeyspace {
/// Returns the underlying LSM-tree's path.
#[must_use]
pub fn path(&self) -> PathBuf {
self.inner.path().into()
}
/// Approximates the number of items in the keyspace.
///
/// For update- or delete-heavy workloads, this value will
/// diverge from the real value, but is a O(1) operation.
///
/// For insert-only workloads (e.g. logs, time series)
/// this value is reliable.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// assert_eq!(tree.approximate_len(), 0);
///
/// tree.insert("1", "abc")?;
/// assert_eq!(tree.approximate_len(), 1);
///
/// tree.remove("1")?;
/// // Oops! approximate_len will not be reliable here
/// assert_eq!(tree.approximate_len(), 2);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[must_use]
pub fn approximate_len(&self) -> usize {
self.inner.approximate_len()
}
/// Removes an item and returns its value if it existed.
///
/// The operation will run wrapped in a transaction.
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let taken = tree.take("a")?.unwrap();
/// assert_eq!(b"abc", &*taken);
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn take<K: Into<UserKey>>(&self, key: K) -> crate::Result<Option<UserValue>> {
self.fetch_update(key, |_| None)
}
/// Atomically updates an item and returns the previous value.
///
/// Returning `None` removes the item if it existed before.
///
/// The operation will run wrapped in a transaction.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, Slice, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let prev = tree.fetch_update("a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"abc", &*prev);
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let prev = tree.fetch_update("a", |_| None)?.unwrap();
/// assert_eq!(b"abc", &*prev);
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn fetch_update<K: Into<UserKey>, F: FnOnce(Option<&UserValue>) -> Option<UserValue>>(
&self,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
let key: UserKey = key.into();
let mut tx = self.db.write_tx();
let prev = tx.fetch_update(self, key, f)?;
tx.commit()?;
Ok(prev)
}
/// Atomically updates an item and returns the new value.
///
/// Returning `None` removes the item if it existed before.
///
/// The operation will run wrapped in a transaction.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, Slice, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let updated = tree.update_fetch("a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"def", &*updated);
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let updated = tree.update_fetch("a", |_| None)?;
/// assert!(updated.is_none());
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn update_fetch<K: Into<UserKey>, F: FnOnce(Option<&UserValue>) -> Option<UserValue>>(
&self,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
let key = key.into();
let mut tx = self.db.write_tx();
let updated = tx.update_fetch(self, key, f)?;
tx.commit()?;
Ok(updated)
}
/// Inserts a key-value pair into the keyspace.
///
/// Keys may be up to 65536 bytes long, values up to 2^32 bytes.
/// Shorter keys and values result in better performance.
///
/// If the key already exists, the item will be overwritten.
///
/// The operation will run wrapped in a transaction.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// assert!(!db.read_tx().is_empty(&tree)?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn insert<K: Into<UserKey>, V: Into<UserValue>>(
&self,
key: K,
value: V,
) -> crate::Result<()> {
let mut tx = self.db.write_tx();
tx.insert(self, key, value);
tx.commit()?;
Ok(())
}
/// Removes an item from the keyspace.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
///
/// The operation will run wrapped in a transaction.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
/// assert!(!db.read_tx().is_empty(&tree)?);
///
/// tree.remove("a")?;
/// assert!(db.read_tx().is_empty(&tree)?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn remove<K: Into<UserKey>>(&self, key: K) -> crate::Result<()> {
let mut tx = self.db.write_tx();
tx.remove(self, key);
tx.commit()?;
Ok(())
}
/// Removes an item from the keyspace, leaving behind a weak tombstone.
///
/// The tombstone marker of this delete operation will vanish when it
/// collides with its corresponding insertion.
/// This may cause older versions of the value to be resurrected, so it should
/// only be used and preferred in scenarios where a key is only ever written once.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
///
/// The operation will run wrapped in a transaction.
///
/// # Experimental
///
/// This function is currently experimental.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
/// assert!(!db.read_tx().is_empty(&tree)?);
///
/// tree.remove_weak("a")?;
/// assert!(db.read_tx().is_empty(&tree)?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
#[doc(hidden)]
pub fn remove_weak<K: Into<UserKey>>(&self, key: K) -> crate::Result<()> {
let mut tx = self.db.write_tx();
tx.remove_weak(self, key);
tx.commit()?;
Ok(())
}
/// Retrieves an item from the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("my_value".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn get<K: AsRef<[u8]>>(&self, key: K) -> crate::Result<Option<lsm_tree::UserValue>> {
self.inner.get(key)
}
/// Retrieves the size of an item from the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
///
/// let len = tree.size_of("a")?.unwrap_or_default();
/// assert_eq!("my_value".len() as u32, len);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn size_of<K: AsRef<[u8]>>(&self, key: K) -> crate::Result<Option<u32>> {
self.inner.size_of(key)
}
/// Returns the first key-value pair in the keyspace.
/// The key in this pair is the minimum key in the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
/// tree.insert("b", "my_value")?;
///
/// assert_eq!(b"a", &*tree.first_key_value().unwrap().key()?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[must_use]
pub fn first_key_value(&self) -> Option<Guard> {
let read_tx = self.db.read_tx();
read_tx.first_key_value(self)
}
/// Returns the last key-value pair in the keyspace.
/// The key in this pair is the maximum key in the keyspace.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
/// tree.insert("b", "my_value")?;
///
/// assert_eq!(b"b", &*tree.last_key_value().unwrap().key()?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[must_use]
pub fn last_key_value(&self) -> Option<Guard> {
let read_tx = self.db.read_tx();
read_tx.last_key_value(self)
}
/// Returns `true` if the keyspace contains the specified key.
///
/// The operation will run wrapped in a read snapshot.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "my_value")?;
///
/// assert!(tree.contains_key("a")?);
/// assert!(!tree.contains_key("b")?);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn contains_key<K: AsRef<[u8]>>(&self, key: K) -> crate::Result<bool> {
self.inner.contains_key(key)
}
/// Allows access to the inner keyspace handle, allowing to
/// escape from the transactional context.
#[doc(hidden)]
#[must_use]
pub fn inner(&self) -> &Keyspace {
&self.inner
}
}
crates/fjall/src/tx/single_writer/mod.rs
-189
View File
@@ -1,189 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
mod keyspace;
mod write_tx;
use crate::{
keyspace::KeyspaceKey, Config, Database, KeyspaceCreateOptions, PersistMode, Snapshot,
};
use std::{
path::Path,
sync::{Arc, Mutex},
};
pub use keyspace::SingleWriterTxKeyspace;
pub use write_tx::WriteTransaction;
pub trait Openable {
fn open(config: Config) -> crate::Result<Self>
where
Self: Sized;
}
/// Single-writer transactional database
#[derive(Clone)]
pub struct TxDatabase {
pub(crate) inner: Database,
single_writer_lock: Arc<Mutex<()>>,
}
impl Openable for TxDatabase {
fn open(config: Config) -> crate::Result<Self>
where
Self: Sized,
{
Ok(Self {
inner: crate::Database::open(config)?,
single_writer_lock: Arc::default(),
})
}
}
impl TxDatabase {
/// Creates a new database builder to create or open a database at `path`.
pub fn builder(path: impl AsRef<Path>) -> crate::DatabaseBuilder<Self> {
crate::DatabaseBuilder::new(path.as_ref())
}
#[doc(hidden)]
#[must_use]
pub fn inner(&self) -> &Database {
&self.inner
}
/// Starts a new writeable transaction.
#[must_use]
#[expect(clippy::missing_panics_doc)]
pub fn write_tx(&self) -> WriteTransaction<'_> {
#[expect(clippy::expect_used)]
let guard = self.single_writer_lock.lock().expect("poisoned tx lock");
let mut write_tx = WriteTransaction::new(
self.clone(),
self.inner.supervisor.snapshot_tracker.open(),
guard,
);
if !self.inner.config.manual_journal_persist {
write_tx = write_tx.durability(Some(PersistMode::Buffer));
}
write_tx
}
/// Starts a new read-only transaction (a.k.a. [`Snapshot`]).
#[must_use]
pub fn read_tx(&self) -> Snapshot {
self.inner.snapshot()
}
/// Flushes the active journal. The durability depends on the [`PersistMode`]
/// used.
///
/// Persisting only affects durability, NOT consistency! Even without flushing
/// data is crash-safe.
///
/// # Examples
///
/// ```
/// # use fjall::{PersistMode, SingleWriterTxDatabase, KeyspaceCreateOptions};
/// # let folder = tempfile::tempdir()?;
/// let db = SingleWriterTxDatabase::builder(folder).open()?;
/// let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?;
///
/// items.insert("a", "hello")?;
///
/// db.persist(PersistMode::SyncAll)?;
/// #
/// # Ok::<_, fjall::Error>(())
/// ```
///
/// # Errors
///
/// Returns error, if an IO error occurred.
pub fn persist(&self, mode: PersistMode) -> crate::Result<()> {
self.inner.persist(mode)
}
/// Creates or opens a keyspace.
///
/// If the keyspace does not yet exist, it will be created configured with `create_options`.
/// Otherwise simply a handle to the existing keyspace will be returned.
///
/// Keyspace names can be up to 255 characters long and can not be empty.
///
/// # Errors
///
/// Returns error, if an IO error occurred.
///
/// # Panics
///
/// Panics if the keyspace name is invalid.
pub fn keyspace(
&self,
name: &str,
create_options: impl FnOnce() -> KeyspaceCreateOptions,
) -> crate::Result<SingleWriterTxKeyspace> {
let keyspace = self.inner.keyspace(name, create_options)?;
Ok(SingleWriterTxKeyspace {
inner: keyspace,
db: self.clone(),
})
}
/// Returns the number of keyspaces.
#[must_use]
pub fn keyspace_count(&self) -> usize {
self.inner.keyspace_count()
}
/// Gets a list of all keyspace names in the database.
#[must_use]
pub fn list_keyspace_names(&self) -> Vec<KeyspaceKey> {
self.inner.list_keyspace_names()
}
/// Returns `true` if the keyspace with the given name exists.
#[must_use]
pub fn keyspace_exists(&self, name: &str) -> bool {
self.inner.keyspace_exists(name)
}
/// Returns the current write buffer size (active + sealed memtables).
#[must_use]
pub fn write_buffer_size(&self) -> u64 {
self.inner.write_buffer_size()
}
/// Returns the number of journal fragments on disk.
#[must_use]
pub fn journal_count(&self) -> usize {
self.inner.journal_count()
}
/// Returns the disk space usage of the entire database.
///
/// # Errors
///
/// Returns error, if an IO error occurred.
pub fn disk_space(&self) -> crate::Result<u64> {
self.inner.disk_space()
}
/// Opens a database in the given directory.
///
/// # Errors
///
/// Returns error, if an IO error occurred.
pub fn open(config: Config) -> crate::Result<Self> {
let inner = Database::create_or_recover(config)?;
Ok(Self {
inner,
single_writer_lock: Arc::default(),
})
}
}
crates/fjall/src/tx/single_writer/write_tx.rs
-375
View File
@@ -1,375 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{
snapshot_nonce::SnapshotNonce,
tx::{single_writer::keyspace::SingleWriterTxKeyspace, write_tx::BaseTransaction},
Guard, Iter, Keyspace, PersistMode, Readable, SingleWriterTxDatabase,
};
use lsm_tree::{UserKey, UserValue};
use std::{ops::RangeBounds, sync::MutexGuard};
/// A single-writer (serialized) cross-keyspace transaction
///
/// Use [`WriteTransaction::commit`] to commit changes to the keyspace(s).
///
/// Transactions keep a consistent view of the database at the time,
/// meaning old data will not be dropped until it is not referenced by any active transaction.
///
/// For that reason, you should try to keep transactions short-lived, and make sure they
/// are not held somewhere forever.
#[clippy::has_significant_drop]
pub struct WriteTransaction<'a> {
_guard: MutexGuard<'a, ()>,
inner: BaseTransaction,
}
impl Readable for WriteTransaction<'_> {
fn get<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<Option<UserValue>> {
self.inner.get(keyspace, key)
}
fn contains_key<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<bool> {
self.inner.contains_key(keyspace, key)
}
fn first_key_value(&self, keyspace: impl AsRef<Keyspace>) -> Option<Guard> {
self.inner.first_key_value(keyspace)
}
fn last_key_value(&self, keyspace: impl AsRef<Keyspace>) -> Option<Guard> {
self.inner.last_key_value(keyspace)
}
fn size_of<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<Option<u32>> {
self.inner.size_of(keyspace, key)
}
fn iter(&self, keyspace: impl AsRef<Keyspace>) -> Iter {
self.inner.iter(keyspace)
}
fn range<K: AsRef<[u8]>, R: RangeBounds<K>>(
&self,
keyspace: impl AsRef<Keyspace>,
range: R,
) -> Iter {
self.inner.range(keyspace, range)
}
fn prefix<K: AsRef<[u8]>>(&self, keyspace: impl AsRef<Keyspace>, prefix: K) -> Iter {
self.inner.prefix(keyspace, prefix)
}
}
impl<'tx> WriteTransaction<'tx> {
pub(crate) fn new(
db: SingleWriterTxDatabase,
nonce: SnapshotNonce,
guard: MutexGuard<'tx, ()>,
) -> Self {
Self {
_guard: guard,
inner: BaseTransaction::new(db.inner, nonce),
}
}
/// Sets the durability level.
#[must_use]
pub fn durability(mut self, mode: Option<PersistMode>) -> Self {
self.inner = self.inner.durability(mode);
self
}
/// Removes an item and returns its value if it existed.
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx();
///
/// let taken = tx.take(&tree, "a")?.unwrap();
/// assert_eq!(b"abc", &*taken);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn take<K: Into<UserKey>>(
&mut self,
keyspace: &SingleWriterTxKeyspace,
key: K,
) -> crate::Result<Option<UserValue>> {
self.inner.take(keyspace.inner(), key)
}
/// Atomically updates an item and returns the new value.
///
/// Returning `None` removes the item if it existed before.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Slice, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx();
///
/// let updated = tx.update_fetch(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"def", &*updated);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx();
///
/// let updated = tx.update_fetch(&tree, "a", |_| None)?;
/// assert!(updated.is_none());
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn update_fetch<K: Into<UserKey>, F: FnOnce(Option<&UserValue>) -> Option<UserValue>>(
&mut self,
keyspace: &SingleWriterTxKeyspace,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
self.inner.update_fetch(keyspace.inner(), key, f)
}
/// Atomically updates an item and returns the previous value.
///
/// Returning `None` removes the item if it existed before.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Slice, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx();
///
/// let prev = tx.fetch_update(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap();
/// assert_eq!(b"abc", &*prev);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert_eq!(Some("def".as_bytes().into()), item);
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// # use std::sync::Arc;
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "abc")?;
///
/// let mut tx = db.write_tx();
///
/// let prev = tx.fetch_update(&tree, "a", |_| None)?.unwrap();
/// assert_eq!(b"abc", &*prev);
/// tx.commit()?;
///
/// let item = tree.get("a")?;
/// assert!(item.is_none());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn fetch_update<K: Into<UserKey>, F: FnOnce(Option<&UserValue>) -> Option<UserValue>>(
&mut self,
keyspace: &SingleWriterTxKeyspace,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
self.inner.fetch_update(keyspace.inner(), key, f)
}
// Inserts a key-value pair into the keyspace.
///
/// Keys may be up to 65536 bytes long, values up to 2^32 bytes.
/// Shorter keys and values result in better performance.
///
/// If the key already exists, the item will be overwritten.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "previous_value")?;
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
///
/// let mut tx = db.write_tx();
/// tx.insert(&tree, "a", "new_value");
///
/// drop(tx);
///
/// // Write was not committed
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
pub fn insert<K: Into<UserKey>, V: Into<UserValue>>(
&mut self,
keyspace: &SingleWriterTxKeyspace,
key: K,
value: V,
) {
self.inner.insert(keyspace.inner(), key, value);
}
/// Removes an item from the keyspace.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "previous_value")?;
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
///
/// let mut tx = db.write_tx();
/// tx.remove(&tree, "a");
///
/// // Read-your-own-write
/// let item = tx.get(&tree, "a")?;
/// assert_eq!(None, item);
///
/// drop(tx);
///
/// // Deletion was not committed
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
pub fn remove<K: Into<UserKey>>(&mut self, keyspace: &SingleWriterTxKeyspace, key: K) {
self.inner.remove(keyspace.inner(), key);
}
/// Removes an item from the keyspace, leaving behind a weak tombstone.
///
/// The tombstone marker of this delete operation will vanish when it
/// collides with its corresponding insertion.
/// This may cause older versions of the value to be resurrected, so it should
/// only be used and preferred in scenarios where a key is only ever written once.
///
/// # Experimental
///
/// This function is currently experimental.
///
/// # Examples
///
/// ```
/// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable};
/// #
/// # let folder = tempfile::tempdir()?;
/// # let db = SingleWriterTxDatabase::builder(folder).open()?;
/// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?;
/// tree.insert("a", "previous_value")?;
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
///
/// let mut tx = db.write_tx();
/// tx.remove_weak(&tree, "a");
///
/// // Read-your-own-write
/// let item = tx.get(&tree, "a")?;
/// assert_eq!(None, item);
///
/// drop(tx);
///
/// // Deletion was not committed
/// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap());
/// #
/// # Ok::<(), fjall::Error>(())
/// ```
#[doc(hidden)]
pub fn remove_weak<K: Into<UserKey>>(&mut self, keyspace: &SingleWriterTxKeyspace, key: K) {
self.inner.remove_weak(keyspace.inner(), key);
}
/// Commits the transaction.
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub fn commit(self) -> crate::Result<()> {
self.inner.commit()
}
/// More explicit alternative to dropping the transaction
/// to roll it back.
pub fn rollback(self) {
self.inner.rollback();
}
}
crates/fjall/src/tx/write_tx.rs
-404
View File
@@ -1,404 +0,0 @@
// Copyright (c) 2024-present, fjall-rs
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use crate::{
batch::item::Item, snapshot_nonce::SnapshotNonce, Database, Guard, HashMap, Iter, Keyspace,
OwnedWriteBatch, PersistMode, Readable,
};
use lsm_tree::{AbstractTree, InternalValue, Memtable, SeqNo, UserKey, UserValue};
use std::{ops::RangeBounds, sync::Arc};
fn ignore_tombstone_value(item: InternalValue) -> Option<InternalValue> {
if item.is_tombstone() {
None
} else {
Some(item)
}
}
pub(super) struct BaseTransaction {
/// Database to work with
pub(super) db: Database,
/// Ephemeral transaction changes
///
/// Used for RYOW (read-your-own-writes)
pub(crate) memtables: HashMap<Keyspace, Arc<Memtable>>,
/// The snapshot, for repeatable reads
pub(crate) nonce: SnapshotNonce,
/// Durability level used, see [`PersistMode`].
pub(crate) durability: Option<PersistMode>,
/// Current sequence number
///
/// The sequence number starts at 0b1000...0000 and is incremented with each write.
///
/// This ensures that writes within the transaction are always newer than any existing data.
pub(crate) seqno: SeqNo,
}
impl Readable for BaseTransaction {
fn get<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<Option<UserValue>> {
let keyspace = keyspace.as_ref();
let key = key.as_ref();
if let Some(memtable) = self.memtables.get(keyspace) {
if let Some(item) = memtable.get(key, SeqNo::MAX) {
return Ok(ignore_tombstone_value(item).map(|x| x.value));
}
}
let res = keyspace.tree.get(key, self.nonce.instant)?;
Ok(res)
}
fn contains_key<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<bool> {
let keyspace = keyspace.as_ref();
let key = key.as_ref();
if let Some(memtable) = self.memtables.get(keyspace) {
if let Some(item) = memtable.get(key, SeqNo::MAX) {
return Ok(!item.key.is_tombstone());
}
}
let contains = keyspace.tree.contains_key(key, self.nonce.instant)?;
Ok(contains)
}
fn first_key_value(&self, keyspace: impl AsRef<Keyspace>) -> Option<Guard> {
self.iter(keyspace.as_ref()).next()
}
fn last_key_value(&self, keyspace: impl AsRef<Keyspace>) -> Option<Guard> {
self.iter(keyspace.as_ref()).next_back()
}
fn size_of<K: AsRef<[u8]>>(
&self,
keyspace: impl AsRef<Keyspace>,
key: K,
) -> crate::Result<Option<u32>> {
let keyspace = keyspace.as_ref();
let key = key.as_ref();
if let Some(memtable) = self.memtables.get(keyspace) {
if let Some(item) = memtable.get(key, SeqNo::MAX) {
// NOTE: Values are limited to u32 in lsm-tree
#[expect(clippy::cast_possible_truncation)]
return Ok(ignore_tombstone_value(item).map(|x| x.value.len() as u32));
}
}
let res = keyspace.tree.size_of(key, self.nonce.instant)?;
Ok(res)
}
fn iter(&self, keyspace: impl AsRef<Keyspace>) -> Iter {
let keyspace = keyspace.as_ref();
let iter = keyspace.tree.iter(
self.nonce.instant,
self.memtables
.get(keyspace)
.cloned()
.map(|mt| (mt, self.seqno)),
);
Iter::new(self.nonce.clone(), iter)
}
fn range<K: AsRef<[u8]>, R: RangeBounds<K>>(
&self,
keyspace: impl AsRef<Keyspace>,
range: R,
) -> Iter {
let keyspace = keyspace.as_ref();
let iter = keyspace.tree.range(
range,
self.nonce.instant,
self.memtables
.get(keyspace)
.cloned()
.map(|mt| (mt, self.seqno)),
);
Iter::new(self.nonce.clone(), iter)
}
fn prefix<K: AsRef<[u8]>>(&self, keyspace: impl AsRef<Keyspace>, prefix: K) -> Iter {
let keyspace = keyspace.as_ref();
let iter = keyspace.tree.prefix(
prefix,
self.nonce.instant,
self.memtables
.get(keyspace)
.cloned()
.map(|mt| (mt, self.seqno)),
);
Iter::new(self.nonce.clone(), iter)
}
}
impl BaseTransaction {
pub(crate) fn new(db: Database, nonce: SnapshotNonce) -> Self {
Self {
db,
memtables: HashMap::default(),
nonce,
durability: None,
seqno: 0x8000_0000_0000_0000,
}
}
/// Sets the durability level.
#[must_use]
pub(super) fn durability(mut self, mode: Option<PersistMode>) -> Self {
self.durability = mode;
self
}
/// Removes an item and returns its value if it existed.
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub(super) fn take<K: Into<UserKey>>(
&mut self,
keyspace: &Keyspace,
key: K,
) -> crate::Result<Option<UserValue>> {
self.fetch_update(keyspace, key, |_| None)
}
/// Atomically updates an item and returns the new value.
///
/// Returning `None` removes the item if it existed before.
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub(super) fn update_fetch<
K: Into<UserKey>,
F: FnOnce(Option<&UserValue>) -> Option<UserValue>,
>(
&mut self,
keyspace: &Keyspace,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
let key = key.into();
let prev = self.get(keyspace, &key)?;
let updated = f(prev.as_ref());
if let Some(value) = updated.clone() {
// NOTE: Skip insert if the value hasn't changed
if prev.as_ref() != Some(&value) {
self.insert(keyspace, key, value);
}
} else if prev.is_some() {
self.remove(keyspace, key);
}
Ok(updated)
}
/// Atomically updates an item and returns the previous value.
///
/// Returning `None` removes the item if it existed before.
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub(super) fn fetch_update<
K: Into<UserKey>,
F: FnOnce(Option<&UserValue>) -> Option<UserValue>,
>(
&mut self,
keyspace: &Keyspace,
key: K,
f: F,
) -> crate::Result<Option<UserValue>> {
let key = key.into();
let prev = self.get(keyspace, &key)?;
let updated = f(prev.as_ref());
if let Some(value) = updated {
// NOTE: Skip insert if the value hasn't changed
if prev.as_ref() != Some(&value) {
self.insert(keyspace, key, value);
}
} else if prev.is_some() {
self.remove(keyspace, key);
}
Ok(prev)
}
/// Inserts a key-value pair into the keyspace.
///
/// Keys may be up to 65536 bytes long, values up to 2^32 bytes.
/// Shorter keys and values result in better performance.
///
/// If the key already exists, the item will be overwritten.
pub(super) fn insert<K: Into<UserKey>, V: Into<UserValue>>(
&mut self,
keyspace: &Keyspace,
key: K,
value: V,
) {
self.memtables
.entry(keyspace.clone())
.or_insert_with(|| Arc::new(Memtable::new(0)))
.insert(lsm_tree::InternalValue::from_components(
key,
value,
self.seqno,
lsm_tree::ValueType::Value,
));
self.seqno += 1;
}
/// Removes an item from the keyspace.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
pub(super) fn remove<K: Into<UserKey>>(&mut self, keyspace: &Keyspace, key: K) {
self.memtables
.entry(keyspace.clone())
.or_insert_with(|| Arc::new(Memtable::new(0)))
.insert(lsm_tree::InternalValue::new_tombstone(key, self.seqno));
self.seqno += 1;
}
/// Removes an item from the keyspace, leaving behind a weak tombstone.
///
/// The tombstone marker of this delete operation will vanish when it
/// collides with its corresponding insertion.
/// This may cause older versions of the value to be resurrected, so it should
/// only be used and preferred in scenarios where a key is only ever written once.
///
/// The key may be up to 65536 bytes long.
/// Shorter keys result in better performance.
pub(super) fn remove_weak<K: Into<UserKey>>(&mut self, keyspace: &Keyspace, key: K) {
self.memtables
.entry(keyspace.clone())
.or_insert_with(|| Arc::new(Memtable::new(0)))
.insert(lsm_tree::InternalValue::new_weak_tombstone(key, self.seqno));
self.seqno += 1;
}
/// Commits the transaction.
///
/// # Errors
///
/// Will return `Err` if an IO error occurs.
pub(super) fn commit(self) -> crate::Result<()> {
// skip all the logic if no keys were written to
if self.memtables.is_empty() {
return Ok(());
}
// TODO: instead of using batch, write batch::commit as a generic function that takes
// a impl Iterator<BatchItem>
// that way, we don't have to move the memtable(s) into the batch first to commit
let mut batch = OwnedWriteBatch::new(self.db).durability(self.durability);
for (keyspace, memtable) in self.memtables {
let mut prev_key: Option<UserKey> = None;
for item in memtable.iter() {
if let Some(prev_key) = &prev_key {
if item.key.user_key == prev_key {
continue;
}
}
batch.data.push(Item::new(
keyspace.clone(),
item.key.user_key.clone(),
item.value.clone(),
item.key.value_type,
));
prev_key = Some(item.key.user_key.clone());
}
}
batch.commit()?;
Ok(())
}
/// More explicit alternative to dropping the transaction
/// to roll it back.
#[expect(clippy::unused_self)]
pub(super) fn rollback(self) {}
}
#[cfg(test)]
mod tests {
use crate::{
tx::{single_writer::SingleWriterTxKeyspace, write_tx::BaseTransaction},
KeyspaceCreateOptions, SingleWriterTxDatabase,
};
use tempfile::TempDir;
struct TestEnv {
db: SingleWriterTxDatabase,
tree: SingleWriterTxKeyspace,
#[expect(unused)]
tmpdir: TempDir,
}
fn setup() -> Result<TestEnv, Box<dyn std::error::Error>> {
let tmpdir: TempDir = tempfile::tempdir()?;
let db = SingleWriterTxDatabase::builder(tmpdir.path()).open()?;
let tree = db.keyspace("foo", KeyspaceCreateOptions::default)?;
Ok(TestEnv { db, tree, tmpdir })
}
#[test]
fn update_fetch() -> Result<(), Box<dyn std::error::Error>> {
let env = setup()?;
env.tree.insert([2u8], [20u8])?;
let mut tx = BaseTransaction::new(
env.db.inner.clone(),
env.db.inner().supervisor.snapshot_tracker.open(),
);
let new = tx.update_fetch(env.tree.inner(), [2u8], |v| {
v.and_then(|v| v.first().copied()).map(|v| [v + 5].into())
})?;
assert_eq!(new, Some([25u8].into()));
tx.commit()?;
Ok(())
}
}
crates/fjall/src/version.rs
+9 -4
View File
@@ -18,6 +18,9 @@ pub enum FormatVersion {
/// Fixed-width key/value lengths in LSM table data blocks
V4,
/// 32-bit LSM tree and table IDs
V5,
}
impl std::fmt::Display for FormatVersion {
@@ -33,6 +36,7 @@ impl From<FormatVersion> for u8 {
FormatVersion::V2 => 2,
FormatVersion::V3 => 3,
FormatVersion::V4 => 4,
FormatVersion::V5 => 5,
}
}
}
@@ -46,6 +50,7 @@ impl TryFrom<u8> for FormatVersion {
2 => Ok(Self::V2),
3 => Ok(Self::V3),
4 => Ok(Self::V4),
5 => Ok(Self::V5),
_ => Err(()),
}
}
@@ -110,13 +115,13 @@ mod tests {
}
#[test]
pub fn version_serialize_and_deserialize_4() -> crate::Result<()> {
pub fn version_serialize_and_deserialize_5() -> crate::Result<()> {
let mut bytes = vec![];
FormatVersion::V4.write_file_header(&mut bytes)?;
assert_eq!(bytes, &[b'F', b'J', b'L', 4]);
FormatVersion::V5.write_file_header(&mut bytes)?;
assert_eq!(bytes, &[b'F', b'J', b'L', 5]);
assert_eq!(
FormatVersion::parse_file_header(&bytes),
Some(FormatVersion::V4),
Some(FormatVersion::V5),
);
Ok(())
}
crates/fjall/src/worker_pool.rs
+3 -3
View File
@@ -3,13 +3,13 @@
// (found in the LICENSE-* files in the repository)
use crate::{
compaction::worker::run as run_compaction, flush::worker::run as run_flush, poison::PoisonDart,
stats::Stats, supervisor::Supervisor, Keyspace,
Keyspace, compaction::worker::run as run_compaction, flush::worker::run as run_flush,
poison::PoisonDart, stats::Stats, supervisor::Supervisor,
};
use lsm_tree::MemtableId;
use std::{
borrow::Cow,
sync::{atomic::AtomicUsize, Arc, Mutex},
sync::{Arc, Mutex, atomic::AtomicUsize},
thread::JoinHandle,
};
crates/fjall/src/write_buffer_manager.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// This source code is licensed under both the Apache 2.0 and MIT License
// (found in the LICENSE-* files in the repository)
use std::sync::{atomic::AtomicU64, Arc};
use std::sync::{Arc, atomic::AtomicU64};
/// Keeps track of the size of the database's write buffer
#[derive(Clone, Default, Debug)]
crates/fjall/test_fixture/v1_keyspace/partitions/a/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/a/levels
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/a/segments/0
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/a/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/b/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/b/levels
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/b/segments/1
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/b/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/c/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/c/levels
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/partitions/c/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/levels
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/segments/0
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/levels
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/segments/1
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/levels
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v1_keyspace_corrupt_journal/version
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/journals/2
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default1/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default1/levels
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default1/manifest
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default1/segments/0
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default2/blobs/.vlog
-1
View File
@@ -1 +0,0 @@
VLG
crates/fjall/test_fixture/v2_keyspace/partitions/default2/blobs/segments/0
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default2/blobs/vlog_manifest
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default2/config
BIN
View File
Binary file not shown.
crates/fjall/test_fixture/v2_keyspace/partitions/default2/levels
BIN
View File
Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More