Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 06:01:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: wip

Browse Source
This commit is contained in:
nym21
2025-06-01 14:37:19 +02:00
parent f976f672cf
commit cbcf603b63
30 changed files with 326 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -1
View File
@@ -8,7 +8,7 @@ target
*\ copy*
# Ignored
/_*
_*
# Editors
.vscode
Cargo.lock
Generated
+1 -15
View File
@@ -437,18 +437,11 @@ dependencies = [
"brk_parser",
"brk_state",
"brk_vec",
"clap",
"clap_derive",
"color-eyre",
"derive_deref",
"fjall",
"jiff",
"log",
"rayon",
"serde",
"serde_json",
"zerocopy",
"zerocopy-derive",
]
[[package]]
@@ -466,9 +459,7 @@ dependencies = [
"rlimit",
"serde",
"serde_bytes",
"serde_derive",
"serde_json",
"thiserror",
"zerocopy",
"zerocopy-derive",
]
@@ -507,12 +498,10 @@ dependencies = [
"brk_parser",
"brk_store",
"brk_vec",
"byteview",
"color-eyre",
"fjall",
"log",
"rayon",
"zerocopy",
]
[[package]]
@@ -593,9 +582,7 @@ version = "0.0.40"
dependencies = [
"brk_core",
"brk_store",
"brk_vec",
"fjall",
"rayon",
"serde",
"zerocopy",
"zerocopy-derive",
@@ -605,11 +592,10 @@ dependencies = [
name = "brk_store"
version = "0.0.40"
dependencies = [
"arc-swap",
"brk_core",
"byteview",
"color-eyre",
"fjall",
"zerocopy",
]
[[package]]
Cargo.toml
+1
View File
@@ -16,6 +16,7 @@ panic = "abort"
inherits = "release"
[workspace.dependencies]
arc-swap = "1.7.1"
axum = "0.8.4"
bitcoin = { version = "0.32.6", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
crates/brk_computer/Cargo.toml
-7
View File
@@ -17,15 +17,8 @@ brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_state = { workspace = true }
brk_vec = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
crates/brk_computer/src/vecs/stateful/cohort.rs
+4 -4
View File
@@ -1,8 +1,6 @@
use std::{fs, path::Path};
use brk_core::{
CheckedSub, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredUsize, Version,
};
use brk_core::{DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredUsize, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_state::CohortState;
@@ -572,7 +570,7 @@ impl Vecs {
self.starting_height = starting_height;
if let Some(prev_height) = starting_height.checked_sub(Height::new(1)) {
if let Some(prev_height) = starting_height.decremented() {
self.state.supply.value = self
.height_to_supply
.into_iter()
@@ -582,6 +580,8 @@ impl Vecs {
.into_iter()
.unwrap_get_inner(prev_height);
self.state.price_to_amount.copy_db_to_puts();
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
self.state.realized.as_mut().unwrap().cap = height_to_realized_cap
.into_iter()
crates/brk_computer/src/vecs/stateful/mod.rs
+22 -16
View File
@@ -1,6 +1,6 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Sats, Version};
use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
@@ -1174,6 +1174,9 @@ impl Vecs {
};
if stateful_starting_height.is_zero() {
info!("Starting processing utxos from the start");
flat_vecs_
.iter_mut()
.try_for_each(|(_, v)| v.state.price_to_amount.reset_partition())?;
}
let starting_height = starting_indexes
.height
@@ -1399,8 +1402,9 @@ impl Vecs {
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height = dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
let date_height_count = dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
let is_date_last_height =
date_first_height + Height::from(*date_height_count) == height;
let is_date_last_height = date_first_height
+ Height::from(date_height_count).decremented().unwrap()
== height;
let date_price = dateindex_to_close_iter
.as_mut()
.map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
@@ -1417,12 +1421,9 @@ impl Vecs {
if height != Height::ZERO && height.unwrap_to_usize() % 100_000 == 0 {
info!("Flushing...");
utxos_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
self.height_to_opreturn_supply.safe_flush(exit)?;
exit.block();
self.flush_vecs(height, exit)?;
exit.release();
}
Ok(())
@@ -1432,13 +1433,7 @@ impl Vecs {
info!("Flushing...");
// Flush rest of values
self.utxos_vecs
.as_mut_vec()
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
self.height_to_opreturn_supply.safe_flush(exit)?;
self.flush_vecs(height, exit)?;
// Save chain state
self.chain_state.truncate_if_needed(Height::ZERO)?;
@@ -1480,6 +1475,17 @@ impl Vecs {
Ok(())
}
fn flush_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
// Flush rest of values
self.utxos_vecs
.as_mut_vec()
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
self.height_to_opreturn_supply.safe_flush(exit)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.utxos_vecs
crates/brk_core/Cargo.toml
-2
View File
@@ -18,9 +18,7 @@ rapidhash = "1.4.0"
rlimit = "0.10.2"
serde = { workspace = true }
serde_bytes = "0.11.17"
serde_derive = { workspace = true }
serde_json = { workspace = true }
thiserror = "2.0.12"
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
crates/brk_core/src/structs/_addressindex.rs
+2 -3
View File
@@ -63,9 +63,8 @@ impl From<AddressIndex> for usize {
}
}
impl TryFrom<ByteView> for AddressIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
impl From<ByteView> for AddressIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
crates/brk_core/src/structs/_addressindexoutputindex.rs
+2 -3
View File
@@ -15,9 +15,8 @@ pub struct AddressIndexOutputIndex {
outputindex: Outputindex,
}
impl TryFrom<ByteView> for AddressIndexOutputIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
impl From<ByteView> for AddressIndexOutputIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
crates/brk_core/src/structs/addressbyteshash.rs
+3 -6
View File
@@ -5,8 +5,6 @@ use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
use super::{AddressBytes, OutputType};
#[derive(
@@ -41,10 +39,9 @@ impl From<[u8; 8]> for AddressBytesHash {
}
}
impl TryFrom<ByteView> for AddressBytesHash {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for AddressBytesHash {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
crates/brk_core/src/structs/blockhashprefix.rs
+4 -5
View File
@@ -3,7 +3,7 @@ use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Error, copy_first_8bytes};
use crate::copy_first_8bytes;
use super::BlockHash;
@@ -35,10 +35,9 @@ impl From<&BlockHash> for BlockHashPrefix {
}
}
impl TryFrom<ByteView> for BlockHashPrefix {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for BlockHashPrefix {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
crates/brk_core/src/structs/dollars.rs
+5 -6
View File
@@ -9,7 +9,7 @@ use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Error, copy_first_8bytes};
use crate::{CheckedSub, copy_first_8bytes};
use super::{Bitcoin, Cents, Close, Sats, StoredF32, StoredF64};
@@ -256,11 +256,10 @@ impl Ord for Dollars {
}
}
impl TryFrom<ByteView> for Dollars {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
let bytes = copy_first_8bytes(&value)?;
Ok(Self::from(f64::from_be_bytes(bytes)))
impl From<ByteView> for Dollars {
fn from(value: ByteView) -> Self {
let bytes = copy_first_8bytes(&value).unwrap();
Self::from(f64::from_be_bytes(bytes))
}
}
crates/brk_core/src/structs/height.rs
+12 -4
View File
@@ -4,12 +4,15 @@ use std::{
};
use bitcoincore_rpc::{Client, RpcApi};
use byteview::ByteView;
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use super::StoredUsize;
#[derive(
Debug,
Clone,
@@ -147,11 +150,17 @@ impl From<u64> for Height {
}
}
impl From<StoredUsize> for Height {
fn from(value: StoredUsize) -> Self {
Self(*value as u32)
}
}
impl From<usize> for Height {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Height> for usize {
fn from(value: Height) -> Self {
value.0 as usize
@@ -189,10 +198,9 @@ impl TryFrom<&std::path::Path> for Height {
}
}
impl TryFrom<byteview::ByteView> for Height {
type Error = crate::Error;
fn try_from(value: byteview::ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for Height {
fn from(value: byteview::ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
crates/brk_core/src/structs/outputtypeindex.rs
+4 -5
View File
@@ -6,7 +6,7 @@ use serde::Serialize;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Error};
use crate::CheckedSub;
#[derive(
Debug,
@@ -82,10 +82,9 @@ impl Add<OutputTypeIndex> for OutputTypeIndex {
Self(self.0 + rhs.0)
}
}
impl TryFrom<ByteView> for OutputTypeIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for OutputTypeIndex {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
impl From<OutputTypeIndex> for ByteView {
crates/brk_core/src/structs/sats.rs
+5 -6
View File
@@ -8,7 +8,7 @@ use byteview::ByteView;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Error, copy_first_8bytes};
use crate::{CheckedSub, copy_first_8bytes};
use super::{Bitcoin, Cents, Dollars, Height};
@@ -189,11 +189,10 @@ impl From<Sats> for u128 {
}
}
impl TryFrom<ByteView> for Sats {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
let bytes = copy_first_8bytes(&value)?;
Ok(Self::from(u64::from_be_bytes(bytes)))
impl From<ByteView> for Sats {
fn from(value: ByteView) -> Self {
let bytes = copy_first_8bytes(&value).unwrap();
Self::from(u64::from_be_bytes(bytes))
}
}
crates/brk_core/src/structs/txidprefix.rs
+4 -5
View File
@@ -3,7 +3,7 @@ use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Error, copy_first_8bytes};
use crate::copy_first_8bytes;
use super::Txid;
@@ -35,10 +35,9 @@ impl From<&Txid> for TxidPrefix {
}
}
impl TryFrom<ByteView> for TxidPrefix {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for TxidPrefix {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
crates/brk_core/src/structs/txindex.rs
+4 -5
View File
@@ -6,7 +6,7 @@ use serde::Serialize;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Error};
use crate::CheckedSub;
use super::StoredU32;
@@ -95,10 +95,9 @@ impl From<TxIndex> for usize {
}
}
impl TryFrom<ByteView> for TxIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for TxIndex {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
impl From<TxIndex> for ByteView {
crates/brk_indexer/Cargo.toml
-2
View File
@@ -15,9 +15,7 @@ brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vec = { workspace = true }
byteview = { workspace = true }
color-eyre = { workspace = true }
fjall = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
zerocopy = { workspace = true }
crates/brk_indexer/src/stores.rs
+1 -7
View File
@@ -26,7 +26,7 @@ impl Stores {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let keyspace = match Self::open_keyspace(path) {
let keyspace = match brk_store::open_keyspace(path) {
Ok(keyspace) => keyspace,
Err(_) => {
fs::remove_dir_all(path)?;
@@ -316,10 +316,4 @@ impl Stores {
self.blockhashprefix_to_height.rotate_memtable();
self.txidprefix_to_txindex.rotate_memtable();
}
fn open_keyspace(path: &Path) -> fjall::Result<TransactionalKeyspace> {
fjall::Config::new(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.open_transactional()
}
}
crates/brk_server/Cargo.toml
+3
View File
@@ -30,3 +30,6 @@ tokio = { workspace = true }
tower-http = { version = "0.6.4", features = ["compression-full", "trace"] }
zip = "4.0.0"
tracing = "0.1.41"
[package.metadata.cargo-machete]
ignored = ["clap"]
crates/brk_state/Cargo.toml
+3 -2
View File
@@ -8,10 +8,11 @@ repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_vec = { workspace = true }
brk_store = { workspace = true }
fjall = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
[package.metadata.cargo-machete]
ignored = ["zerocopy"]
crates/brk_state/src/cohort.rs
+32 -18
View File
@@ -30,18 +30,12 @@ impl CohortState {
keyspace,
path,
&format!("{name}_price_to_amount"),
version + Version::TWO,
version + Version::new(3),
Some(None),
)?,
})
}
pub fn commit(&mut self, height: Height) -> Result<()> {
self.price_to_amount
.retain_or_del(|_, sats| *sats != Sats::ZERO);
self.price_to_amount.commit(height)
}
pub fn reset_single_iteration_values(&mut self) {
if let Some(realized) = self.realized.as_mut() {
realized.reset_single_iteration_values();
@@ -53,7 +47,7 @@ impl CohortState {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.increment(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) += supply_state.value;
*self.price_to_amount.puts_entry_or_default(&price) += supply_state.value;
}
}
@@ -62,7 +56,7 @@ impl CohortState {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.decrement(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) -= supply_state.value;
*self.price_to_amount.puts_entry_or_default(&price) -= supply_state.value;
}
}
@@ -71,7 +65,7 @@ impl CohortState {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.receive(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) += supply_state.value;
*self.price_to_amount.puts_entry_or_default(&price) += supply_state.value;
}
}
@@ -87,7 +81,7 @@ impl CohortState {
let current_price = current_price.unwrap();
let prev_price = prev_price.unwrap();
realized.send(supply_state, current_price, prev_price, older_than_hour);
*self.price_to_amount.get_mut_or_default(&prev_price) -= supply_state.value;
*self.price_to_amount.puts_entry_or_default(&prev_price) -= supply_state.value;
}
}
@@ -114,21 +108,32 @@ impl CohortState {
}
Ordering::Less => {
state.supply_in_profit += sats;
// if price > Dollars::ZERO {
// state.unrealized_profit +=
// current_price.checked_sub(price).unwrap() * sats;
// }
if price > Dollars::ZERO && current_price > Dollars::ZERO {
let diff = current_price.checked_sub(price).unwrap();
if diff <= Dollars::ZERO {
dbg!(price, current_price, diff, sats);
panic!();
}
state.unrealized_profit += diff * sats;
}
}
Ordering::Greater => {
state.supply_in_loss += sats;
// state.unrealized_loss += price.checked_sub(current_price).unwrap() * sats;
if price > Dollars::ZERO && current_price > Dollars::ZERO {
let diff = price.checked_sub(current_price).unwrap();
if diff <= Dollars::ZERO {
dbg!(price, current_price, diff, sats);
panic!();
}
state.unrealized_loss += diff * sats;
}
}
}
};
self.price_to_amount
.unordered_clone_iter()
.for_each(|(price, sats)| {
.puts_iter()
.for_each(|(&price, &sats)| {
update_state(price, height_price, sats, &mut height_unrealized_state);
if let Some(date_price) = date_price {
@@ -143,4 +148,13 @@ impl CohortState {
(height_unrealized_state, date_unrealized_state)
}
pub fn commit(&mut self, height: Height) -> Result<()> {
self.price_to_amount
.retain_or_del(|_, sats| *sats > Sats::ZERO);
let price_to_amount_puts = self.price_to_amount.clone_puts();
self.price_to_amount.commit(height)?;
self.price_to_amount.append_puts(price_to_amount_puts);
Ok(())
}
}
crates/brk_store/Cargo.toml
+1 -2
View File
@@ -9,8 +9,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
arc-swap = { workspace = true }
brk_core = { workspace = true }
byteview = { workspace = true }
color-eyre = { workspace = true }
fjall = { workspace = true }
zerocopy = { workspace = true }
crates/brk_store/README.md
+1
View File
@@ -0,0 +1 @@
# BRK Store
crates/brk_store/examples/main.rs
+43
View File
@@ -0,0 +1,43 @@
use std::path::Path;
use brk_core::{Dollars, Height, Result, Sats, Version};
use brk_store::Store;
fn main() -> Result<()> {
let p = Path::new("./examples/_fjall");
let keyspace = brk_store::open_keyspace(p)?;
let mut store: Store<Dollars, Sats> =
brk_store::Store::import(&keyspace, p, "n", Version::ZERO, None)?;
store.copy_db_to_puts();
*store.puts_entry_or_default(&Dollars::from(10.0)) += Sats::ONE_BTC;
*store.puts_entry_or_default(&Dollars::from(1.0)) += Sats::ONE_BTC;
*store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC;
*store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC;
dbg!(store.tx_iter().collect::<Vec<_>>());
store.commit(Height::ZERO)?;
store.copy_db_to_puts();
dbg!(store.tx_iter().collect::<Vec<_>>());
*store.puts_entry_or_default(&Dollars::from(10.0)) += Sats::ONE_BTC;
*store.puts_entry_or_default(&Dollars::from(1.0)) += Sats::ONE_BTC;
*store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC;
*store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC;
dbg!(store.tx_iter().collect::<Vec<_>>());
store.commit(Height::from(1_u32))?;
store.copy_db_to_puts();
dbg!(store.tx_iter().collect::<Vec<_>>());
Ok(())
}
crates/brk_store/src/lib.rs
+101 -49
View File
@@ -1,18 +1,23 @@
#![doc = include_str!("../README.md")]
#![doc = "\n## Example\n\n```rust"]
#![doc = include_str!("../examples/main.rs")]
#![doc = "```"]
use std::{
collections::{BTreeMap, BTreeSet},
error,
fmt::Debug,
mem,
path::Path,
sync::Arc,
};
use arc_swap::ArcSwap;
use brk_core::{Height, Result, Value, Version};
use byteview::ByteView;
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace,
TransactionalPartitionHandle,
};
use zerocopy::{Immutable, IntoBytes};
mod meta;
use meta::*;
@@ -21,7 +26,7 @@ pub struct Store<Key, Value> {
meta: StoreMeta,
name: String,
keyspace: TransactionalKeyspace,
partition: TransactionalPartitionHandle,
partition: Arc<ArcSwap<TransactionalPartitionHandle>>,
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
@@ -30,15 +35,20 @@ pub struct Store<Key, Value> {
/// Use default if will read
const DEFAULT_BLOOM_FILTER_BITS: Option<u8> = Some(5);
const CHECK_COLLISISONS: bool = true;
// const CHECK_COLLISISONS: bool = true;
const MAJOR_FJALL_VERSION: Version = Version::TWO;
impl<K, V> Store<K, V>
pub fn open_keyspace(path: &Path) -> fjall::Result<TransactionalKeyspace> {
fjall::Config::new(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.open_transactional()
}
impl<'a, K, V> Store<K, V>
where
K: Debug + Clone + Into<ByteView> + TryFrom<ByteView> + Ord + Immutable + IntoBytes,
V: Debug + Clone + Into<ByteView> + TryFrom<ByteView>,
<K as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
K: Debug + Clone + From<ByteView> + Ord + 'a,
V: Debug + Clone + From<ByteView>,
ByteView: From<K> + From<&'a K> + From<V>,
{
pub fn import(
keyspace: &TransactionalKeyspace,
@@ -65,7 +75,7 @@ where
meta,
name: name.to_owned(),
keyspace: keyspace.clone(),
partition,
partition: Arc::new(ArcSwap::from_pointee(partition)),
rtx,
puts: BTreeMap::new(),
dels: BTreeSet::new(),
@@ -73,39 +83,70 @@ where
})
}
pub fn get(&self, key: &K) -> color_eyre::Result<Option<Value<V>>> {
pub fn get(&self, key: &'a K) -> Result<Option<Value<V>>> {
if let Some(v) = self.puts.get(key) {
Ok(Some(Value::Ref(v)))
} else if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes())? {
Ok(Some(Value::Owned(V::try_from(slice.as_bytes().into())?)))
} else if let Some(slice) = self.rtx.get(&self.partition.load(), ByteView::from(key))? {
Ok(Some(Value::Owned(V::from(ByteView::from(slice)))))
} else {
Ok(None)
}
}
pub fn get_mut_or_default(&mut self, key: &K) -> &mut V
pub fn first_key_value(&self) -> Result<Option<(K, V)>> {
Ok(self
.rtx
.first_key_value(&self.partition.load())?
.map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))))
}
pub fn last_key_value(&self) -> Result<Option<(K, V)>> {
Ok(self
.rtx
.last_key_value(&self.partition.load())?
.map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))))
}
pub fn puts_entry_or_default(&mut self, key: &'a K) -> &mut V
where
V: Default,
{
self.puts.entry(key.clone()).or_insert_with(|| {
if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes()).unwrap() {
V::try_from(slice.as_bytes().into()).unwrap()
} else {
V::default()
}
})
self.puts.entry(key.clone()).or_default()
}
pub fn unordered_clone_iter(&self) -> impl Iterator<Item = (K, V)> {
pub fn tx_iter(&self) -> impl Iterator<Item = (K, V)> {
self.rtx
.iter(&self.partition)
.iter(&self.partition.load())
.map(|res| res.unwrap())
.map(|(k, v)| (K::try_from(ByteView::from(k)).unwrap(), v))
.filter(|(k, _)| !self.puts.contains_key(k) && !self.dels.contains(k))
.map(|(k, v)| (k, V::try_from(ByteView::from(v)).unwrap()))
.chain(self.puts.iter().map(|(k, v)| (k.clone(), v.clone())))
.map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v))))
}
pub fn puts_iter(&self) -> impl Iterator<Item = (&K, &V)> {
self.puts.iter()
}
pub fn clone_puts(&self) -> BTreeMap<K, V> {
self.puts.clone()
}
pub fn append_puts(&mut self, mut other: BTreeMap<K, V>) {
self.puts.append(&mut other);
}
pub fn copy_db_to_puts(&mut self) {
self.append_puts(self.tx_iter().collect());
}
// pub fn unordered_clone_iter(&self) -> impl Iterator<Item = (K, V)> {
// self.rtx
// .keys(&self.partition.load())
// .map(|res| res.unwrap())
// .map(|k| K::from(ByteView::from(k)))
// .filter(|k| !self.puts.contains_key(k) && !self.dels.contains(k))
// .map(|k| (k, self.rtx.get(partition, key) V::from(ByteView::from(v))))
// .chain(self.puts.iter().map(|(k, v)| (k.clone(), v.clone())))
// }
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
if !self.dels.is_empty() {
@@ -153,30 +194,28 @@ where
let mut wtx = self.keyspace.write_tx();
let partition = &self.partition.load();
mem::take(&mut self.dels)
.into_iter()
.for_each(|key| wtx.remove(&self.partition, key.as_bytes()));
.for_each(|key| wtx.remove(partition, ByteView::from(key)));
mem::take(&mut self.puts)
.into_iter()
.for_each(|(key, value)| {
if CHECK_COLLISISONS {
#[allow(unused_must_use)]
if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) {
dbg!(
&key,
V::try_from(value.as_bytes().into()).unwrap(),
&self.meta,
self.rtx.get(&self.partition, key.as_bytes())
);
unreachable!();
}
}
wtx.insert(
&self.partition,
key.as_bytes(),
&*ByteView::try_from(value).unwrap(),
)
// if CHECK_COLLISISONS {
// #[allow(unused_must_use)]
// if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) {
// dbg!(
// &key,
// V::try_from(value.as_bytes().into()).unwrap(),
// &self.meta,
// self.rtx.get(&self.partition, key.as_bytes())
// );
// unreachable!();
// }
// }
wtx.insert(partition, ByteView::from(key), ByteView::from(value))
});
wtx.commit()?;
@@ -187,7 +226,7 @@ where
}
pub fn rotate_memtable(&self) {
let _ = self.partition.inner().rotate_memtable();
let _ = self.partition.load().inner().rotate_memtable();
}
pub fn height(&self) -> Option<Height> {
@@ -225,10 +264,23 @@ where
}
pub fn reset_partition(&mut self) -> Result<()> {
self.keyspace.delete_partition(self.partition.clone())?;
let partition = Arc::try_unwrap(self.partition.swap(unsafe {
#[allow(clippy::uninit_assumed_init, invalid_value)]
mem::MaybeUninit::uninit().assume_init()
}))
.ok()
.unwrap();
self.keyspace.delete_partition(partition)?;
self.keyspace.persist(PersistMode::SyncAll)?;
self.partition =
Self::open_partition_handle(&self.keyspace, &self.name, self.bloom_filter_bits)?;
self.partition.store(Arc::new(Self::open_partition_handle(
&self.keyspace,
&self.name,
self.bloom_filter_bits,
)?));
Ok(())
}
}
crates/brk_store/src/meta.rs
+7 -8
View File
@@ -3,9 +3,8 @@ use std::{
path::{Path, PathBuf},
};
use brk_core::{Result, Version};
use brk_core::{Result, Version, copy_first_8bytes};
use fjall::{TransactionalKeyspace, TransactionalPartitionHandle};
use zerocopy::{FromBytes, IntoBytes};
use super::Height;
@@ -48,7 +47,7 @@ impl StoreMeta {
pathbuf: path.to_owned(),
version,
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
len: Self::read_length_(path)?,
len: Self::read_length_(path),
};
slf.version.write(&slf.path_version())?;
@@ -109,16 +108,16 @@ impl StoreMeta {
path.join("height")
}
fn read_length_(path: &Path) -> Result<usize> {
Ok(fs::read(Self::path_length(path))
.map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default())
.unwrap_or_default())
fn read_length_(path: &Path) -> usize {
fs::read(Self::path_length(path))
.map(|v| usize::from_ne_bytes(copy_first_8bytes(v.as_slice()).unwrap_or_default()))
.unwrap_or_default()
}
fn write_length(&self) -> io::Result<()> {
Self::write_length_(&self.pathbuf, self.len)
}
fn write_length_(path: &Path, len: usize) -> io::Result<()> {
fs::write(Self::path_length(path), len.as_bytes())
fs::write(Self::path_length(path), len.to_ne_bytes())
}
fn path_length(path: &Path) -> PathBuf {
path.join("length")
crates/brk_vec/Cargo.toml
+4 -1
View File
@@ -9,7 +9,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
arc-swap = "1.7.1"
arc-swap = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
clap = { workspace = true }
@@ -22,3 +22,6 @@ serde_json = { workspace = true }
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
zstd = "0.13.3"
[package.metadata.cargo-machete]
ignored = ["clap"]
websites/kibo.money/scripts/main.js
+3
View File
@@ -702,6 +702,9 @@ function createUtils() {
(!unit || thoroughUnitCheck) &&
(id.includes("in-sats") ||
id.endsWith("supply") ||
id.endsWith("supply-even") ||
id.endsWith("supply-in-profit") ||
id.endsWith("supply-in-loss") ||
id.endsWith("stack") ||
(id.endsWith("value") && !id.includes("realized")) ||
((id.includes("coinbase") ||
websites/kibo.money/scripts/options.js
+53 -1
View File
@@ -1154,8 +1154,27 @@ function createPartialOptions(colors) {
createBaseSeries({
key: `${key}supply`,
name: useGroupName ? name : "Supply",
color: color,
color: "list" in args ? color : colors.default,
}),
...(!("list" in args)
? [
createBaseSeries({
key: `${key}supply-in-profit`,
name: useGroupName ? name : "In Profit",
color: colors.green,
}),
createBaseSeries({
key: `${key}supply-in-loss`,
name: useGroupName ? name : "In Loss",
color: colors.red,
}),
createBaseSeries({
key: `${key}supply-even`,
name: useGroupName ? name : "Even",
color: colors.yellow,
}),
]
: []),
createBaseSeries({
key: `${key}supply-in-btc`,
name: useGroupName ? name : "Supply",
@@ -1346,6 +1365,39 @@ function createPartialOptions(colors) {
},
],
},
{
name: "Unrealized",
tree: [
{
name: "profit",
title: `${args.title} Unrealized Profit`,
bottom: list.flatMap(({ color, name, key: _key }) => {
const key = fixKey(_key);
return /** @type {const} */ ([
createBaseSeries({
key: `${key}unrealized-profit`,
name: useGroupName ? name : "Profit",
color: useGroupName ? color : colors.green,
}),
]);
}),
},
{
name: "loss",
title: `${args.title} Unrealized Loss`,
bottom: list.flatMap(({ color, name, key: _key }) => {
const key = fixKey(_key);
return /** @type {const} */ ([
createBaseSeries({
key: `${key}unrealized-loss`,
name: useGroupName ? name : "Loss",
color: useGroupName ? color : colors.red,
}),
]);
}),
},
],
},
],
});
}