diff --git a/.gitignore b/.gitignore index ce5f5f453..5fd6c12b3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ target *\ copy* # Ignored -/_* +_* # Editors .vscode diff --git a/Cargo.lock b/Cargo.lock index c006a147c..d1956e9a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -437,18 +437,11 @@ dependencies = [ "brk_parser", "brk_state", "brk_vec", - "clap", - "clap_derive", "color-eyre", - "derive_deref", "fjall", "jiff", "log", "rayon", - "serde", - "serde_json", - "zerocopy", - "zerocopy-derive", ] [[package]] @@ -466,9 +459,7 @@ dependencies = [ "rlimit", "serde", "serde_bytes", - "serde_derive", "serde_json", - "thiserror", "zerocopy", "zerocopy-derive", ] @@ -507,12 +498,10 @@ dependencies = [ "brk_parser", "brk_store", "brk_vec", - "byteview", "color-eyre", "fjall", "log", "rayon", - "zerocopy", ] [[package]] @@ -593,9 +582,7 @@ version = "0.0.40" dependencies = [ "brk_core", "brk_store", - "brk_vec", "fjall", - "rayon", "serde", "zerocopy", "zerocopy-derive", @@ -605,11 +592,10 @@ dependencies = [ name = "brk_store" version = "0.0.40" dependencies = [ + "arc-swap", "brk_core", "byteview", - "color-eyre", "fjall", - "zerocopy", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 227a7387c..50381d0a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ panic = "abort" inherits = "release" [workspace.dependencies] +arc-swap = "1.7.1" axum = "0.8.4" bitcoin = { version = "0.32.6", features = ["serde"] } bitcoincore-rpc = "0.19.0" diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 336a2320e..edef54911 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -17,15 +17,8 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_state = { workspace = true } brk_vec = { workspace = true } -clap = { workspace = true } -clap_derive = { workspace = true } color-eyre = { workspace = true } -derive_deref = { workspace = true } fjall = { workspace = true } jiff = { workspace = true } log = { workspace = true } rayon = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -zerocopy = { workspace = true } -zerocopy-derive = { workspace = true } diff --git a/crates/brk_computer/src/vecs/stateful/cohort.rs b/crates/brk_computer/src/vecs/stateful/cohort.rs index 1b8972958..32a88299a 100644 --- a/crates/brk_computer/src/vecs/stateful/cohort.rs +++ b/crates/brk_computer/src/vecs/stateful/cohort.rs @@ -1,8 +1,6 @@ use std::{fs, path::Path}; -use brk_core::{ - CheckedSub, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredUsize, Version, -}; +use brk_core::{DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredUsize, Version}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_state::CohortState; @@ -572,7 +570,7 @@ impl Vecs { self.starting_height = starting_height; - if let Some(prev_height) = starting_height.checked_sub(Height::new(1)) { + if let Some(prev_height) = starting_height.decremented() { self.state.supply.value = self .height_to_supply .into_iter() @@ -582,6 +580,8 @@ impl Vecs { .into_iter() .unwrap_get_inner(prev_height); + self.state.price_to_amount.copy_db_to_puts(); + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { self.state.realized.as_mut().unwrap().cap = height_to_realized_cap .into_iter() diff --git a/crates/brk_computer/src/vecs/stateful/mod.rs b/crates/brk_computer/src/vecs/stateful/mod.rs index 6423cc9ac..2eb897389 100644 --- a/crates/brk_computer/src/vecs/stateful/mod.rs +++ b/crates/brk_computer/src/vecs/stateful/mod.rs @@ -1,6 +1,6 @@ use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread}; -use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Sats, Version}; +use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Result, Sats, Version}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{ @@ -1174,6 +1174,9 @@ impl Vecs { }; if stateful_starting_height.is_zero() { info!("Starting processing utxos from the start"); + flat_vecs_ + .iter_mut() + .try_for_each(|(_, v)| v.state.price_to_amount.reset_partition())?; } let starting_height = starting_indexes .height @@ -1399,8 +1402,9 @@ impl Vecs { let dateindex = DateIndex::try_from(date).unwrap(); let date_first_height = dateindex_to_first_height_iter.unwrap_get_inner(dateindex); let date_height_count = dateindex_to_height_count_iter.unwrap_get_inner(dateindex); - let is_date_last_height = - date_first_height + Height::from(*date_height_count) == height; + let is_date_last_height = date_first_height + + Height::from(date_height_count).decremented().unwrap() + == height; let date_price = dateindex_to_close_iter .as_mut() .map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex))); @@ -1417,12 +1421,9 @@ impl Vecs { if height != Height::ZERO && height.unwrap_to_usize() % 100_000 == 0 { info!("Flushing..."); - - utxos_vecs - .par_iter_mut() - .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; - self.height_to_unspendable_supply.safe_flush(exit)?; - self.height_to_opreturn_supply.safe_flush(exit)?; + exit.block(); + self.flush_vecs(height, exit)?; + exit.release(); } Ok(()) @@ -1432,13 +1433,7 @@ impl Vecs { info!("Flushing..."); - // Flush rest of values - self.utxos_vecs - .as_mut_vec() - .par_iter_mut() - .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; - self.height_to_unspendable_supply.safe_flush(exit)?; - self.height_to_opreturn_supply.safe_flush(exit)?; + self.flush_vecs(height, exit)?; // Save chain state self.chain_state.truncate_if_needed(Height::ZERO)?; @@ -1480,6 +1475,17 @@ impl Vecs { Ok(()) } + fn flush_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + // Flush rest of values + self.utxos_vecs + .as_mut_vec() + .par_iter_mut() + .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; + self.height_to_unspendable_supply.safe_flush(exit)?; + self.height_to_opreturn_supply.safe_flush(exit)?; + Ok(()) + } + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ self.utxos_vecs diff --git a/crates/brk_core/Cargo.toml b/crates/brk_core/Cargo.toml index bbcd9c678..186f7a6d5 100644 --- a/crates/brk_core/Cargo.toml +++ b/crates/brk_core/Cargo.toml @@ -18,9 +18,7 @@ rapidhash = "1.4.0" rlimit = "0.10.2" serde = { workspace = true } serde_bytes = "0.11.17" -serde_derive = { workspace = true } serde_json = { workspace = true } -thiserror = "2.0.12" zerocopy = { workspace = true } zerocopy-derive = { workspace = true } diff --git a/crates/brk_core/src/structs/_addressindex.rs b/crates/brk_core/src/structs/_addressindex.rs index b705719e5..bde3019d4 100644 --- a/crates/brk_core/src/structs/_addressindex.rs +++ b/crates/brk_core/src/structs/_addressindex.rs @@ -63,9 +63,8 @@ impl From for usize { } } -impl TryFrom for AddressIndex { - type Error = Error; - fn try_from(value: ByteView) -> Result { +impl From for AddressIndex { + fn from(value: ByteView) -> Self { Ok(Self::read_from_bytes(&value)?) } } diff --git a/crates/brk_core/src/structs/_addressindexoutputindex.rs b/crates/brk_core/src/structs/_addressindexoutputindex.rs index 7a7147ef4..f37bb50ec 100644 --- a/crates/brk_core/src/structs/_addressindexoutputindex.rs +++ b/crates/brk_core/src/structs/_addressindexoutputindex.rs @@ -15,9 +15,8 @@ pub struct AddressIndexOutputIndex { outputindex: Outputindex, } -impl TryFrom for AddressIndexOutputIndex { - type Error = Error; - fn try_from(value: ByteView) -> Result { +impl From for AddressIndexOutputIndex { + fn from(value: ByteView) -> Self { Ok(Self::read_from_bytes(&value)?) } } diff --git a/crates/brk_core/src/structs/addressbyteshash.rs b/crates/brk_core/src/structs/addressbyteshash.rs index 35bcb19d1..0f15c458a 100644 --- a/crates/brk_core/src/structs/addressbyteshash.rs +++ b/crates/brk_core/src/structs/addressbyteshash.rs @@ -5,8 +5,6 @@ use derive_deref::Deref; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::Error; - use super::{AddressBytes, OutputType}; #[derive( @@ -41,10 +39,9 @@ impl From<[u8; 8]> for AddressBytesHash { } } -impl TryFrom for AddressBytesHash { - type Error = Error; - fn try_from(value: ByteView) -> Result { - Ok(Self::read_from_bytes(&value)?) +impl From for AddressBytesHash { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() } } diff --git a/crates/brk_core/src/structs/blockhashprefix.rs b/crates/brk_core/src/structs/blockhashprefix.rs index 6b7d30d19..071f94bc6 100644 --- a/crates/brk_core/src/structs/blockhashprefix.rs +++ b/crates/brk_core/src/structs/blockhashprefix.rs @@ -3,7 +3,7 @@ use derive_deref::Deref; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{Error, copy_first_8bytes}; +use crate::copy_first_8bytes; use super::BlockHash; @@ -35,10 +35,9 @@ impl From<&BlockHash> for BlockHashPrefix { } } -impl TryFrom for BlockHashPrefix { - type Error = Error; - fn try_from(value: ByteView) -> Result { - Ok(Self::read_from_bytes(&value)?) +impl From for BlockHashPrefix { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() } } diff --git a/crates/brk_core/src/structs/dollars.rs b/crates/brk_core/src/structs/dollars.rs index f79a4e4bf..15c88d696 100644 --- a/crates/brk_core/src/structs/dollars.rs +++ b/crates/brk_core/src/structs/dollars.rs @@ -9,7 +9,7 @@ use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Error, copy_first_8bytes}; +use crate::{CheckedSub, copy_first_8bytes}; use super::{Bitcoin, Cents, Close, Sats, StoredF32, StoredF64}; @@ -256,11 +256,10 @@ impl Ord for Dollars { } } -impl TryFrom for Dollars { - type Error = Error; - fn try_from(value: ByteView) -> Result { - let bytes = copy_first_8bytes(&value)?; - Ok(Self::from(f64::from_be_bytes(bytes))) +impl From for Dollars { + fn from(value: ByteView) -> Self { + let bytes = copy_first_8bytes(&value).unwrap(); + Self::from(f64::from_be_bytes(bytes)) } } diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index c66d02bf4..6dec2a44d 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -4,12 +4,15 @@ use std::{ }; use bitcoincore_rpc::{Client, RpcApi}; +use byteview::ByteView; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; +use super::StoredUsize; + #[derive( Debug, Clone, @@ -147,11 +150,17 @@ impl From for Height { } } +impl From for Height { + fn from(value: StoredUsize) -> Self { + Self(*value as u32) + } +} impl From for Height { fn from(value: usize) -> Self { Self(value as u32) } } + impl From for usize { fn from(value: Height) -> Self { value.0 as usize @@ -189,10 +198,9 @@ impl TryFrom<&std::path::Path> for Height { } } -impl TryFrom for Height { - type Error = crate::Error; - fn try_from(value: byteview::ByteView) -> Result { - Ok(Self::read_from_bytes(&value)?) +impl From for Height { + fn from(value: byteview::ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() } } diff --git a/crates/brk_core/src/structs/outputtypeindex.rs b/crates/brk_core/src/structs/outputtypeindex.rs index 523c61213..1befa99ca 100644 --- a/crates/brk_core/src/structs/outputtypeindex.rs +++ b/crates/brk_core/src/structs/outputtypeindex.rs @@ -6,7 +6,7 @@ use serde::Serialize; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Error}; +use crate::CheckedSub; #[derive( Debug, @@ -82,10 +82,9 @@ impl Add for OutputTypeIndex { Self(self.0 + rhs.0) } } -impl TryFrom for OutputTypeIndex { - type Error = Error; - fn try_from(value: ByteView) -> Result { - Ok(Self::read_from_bytes(&value)?) +impl From for OutputTypeIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() } } impl From for ByteView { diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs index 9bfc05dc9..555880824 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -8,7 +8,7 @@ use byteview::ByteView; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Error, copy_first_8bytes}; +use crate::{CheckedSub, copy_first_8bytes}; use super::{Bitcoin, Cents, Dollars, Height}; @@ -189,11 +189,10 @@ impl From for u128 { } } -impl TryFrom for Sats { - type Error = Error; - fn try_from(value: ByteView) -> Result { - let bytes = copy_first_8bytes(&value)?; - Ok(Self::from(u64::from_be_bytes(bytes))) +impl From for Sats { + fn from(value: ByteView) -> Self { + let bytes = copy_first_8bytes(&value).unwrap(); + Self::from(u64::from_be_bytes(bytes)) } } diff --git a/crates/brk_core/src/structs/txidprefix.rs b/crates/brk_core/src/structs/txidprefix.rs index 7107090ab..0db2299da 100644 --- a/crates/brk_core/src/structs/txidprefix.rs +++ b/crates/brk_core/src/structs/txidprefix.rs @@ -3,7 +3,7 @@ use derive_deref::Deref; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{Error, copy_first_8bytes}; +use crate::copy_first_8bytes; use super::Txid; @@ -35,10 +35,9 @@ impl From<&Txid> for TxidPrefix { } } -impl TryFrom for TxidPrefix { - type Error = Error; - fn try_from(value: ByteView) -> Result { - Ok(Self::read_from_bytes(&value)?) +impl From for TxidPrefix { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() } } diff --git a/crates/brk_core/src/structs/txindex.rs b/crates/brk_core/src/structs/txindex.rs index 6bc798a4f..07cac2a5a 100644 --- a/crates/brk_core/src/structs/txindex.rs +++ b/crates/brk_core/src/structs/txindex.rs @@ -6,7 +6,7 @@ use serde::Serialize; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Error}; +use crate::CheckedSub; use super::StoredU32; @@ -95,10 +95,9 @@ impl From for usize { } } -impl TryFrom for TxIndex { - type Error = Error; - fn try_from(value: ByteView) -> Result { - Ok(Self::read_from_bytes(&value)?) +impl From for TxIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() } } impl From for ByteView { diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index db98f0b08..27da1655b 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -15,9 +15,7 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_store = { workspace = true } brk_vec = { workspace = true } -byteview = { workspace = true } color-eyre = { workspace = true } fjall = { workspace = true } log = { workspace = true } rayon = { workspace = true } -zerocopy = { workspace = true } diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index 33ca8992c..7b15a3639 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -26,7 +26,7 @@ impl Stores { pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result { fs::create_dir_all(path)?; - let keyspace = match Self::open_keyspace(path) { + let keyspace = match brk_store::open_keyspace(path) { Ok(keyspace) => keyspace, Err(_) => { fs::remove_dir_all(path)?; @@ -316,10 +316,4 @@ impl Stores { self.blockhashprefix_to_height.rotate_memtable(); self.txidprefix_to_txindex.rotate_memtable(); } - - fn open_keyspace(path: &Path) -> fjall::Result { - fjall::Config::new(path.join("fjall")) - .max_write_buffer_size(32 * 1024 * 1024) - .open_transactional() - } } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index ef75e75d2..3196206ce 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -30,3 +30,6 @@ tokio = { workspace = true } tower-http = { version = "0.6.4", features = ["compression-full", "trace"] } zip = "4.0.0" tracing = "0.1.41" + +[package.metadata.cargo-machete] +ignored = ["clap"] diff --git a/crates/brk_state/Cargo.toml b/crates/brk_state/Cargo.toml index a0f130a73..b14ff8804 100644 --- a/crates/brk_state/Cargo.toml +++ b/crates/brk_state/Cargo.toml @@ -8,10 +8,11 @@ repository.workspace = true [dependencies] brk_core = { workspace = true } -brk_vec = { workspace = true } brk_store = { workspace = true } fjall = { workspace = true } -rayon = { workspace = true } serde = { workspace = true } zerocopy = { workspace = true } zerocopy-derive = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["zerocopy"] diff --git a/crates/brk_state/src/cohort.rs b/crates/brk_state/src/cohort.rs index e0924e404..8e6e4f10b 100644 --- a/crates/brk_state/src/cohort.rs +++ b/crates/brk_state/src/cohort.rs @@ -30,18 +30,12 @@ impl CohortState { keyspace, path, &format!("{name}_price_to_amount"), - version + Version::TWO, + version + Version::new(3), Some(None), )?, }) } - pub fn commit(&mut self, height: Height) -> Result<()> { - self.price_to_amount - .retain_or_del(|_, sats| *sats != Sats::ZERO); - self.price_to_amount.commit(height) - } - pub fn reset_single_iteration_values(&mut self) { if let Some(realized) = self.realized.as_mut() { realized.reset_single_iteration_values(); @@ -53,7 +47,7 @@ impl CohortState { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.increment(supply_state, price); - *self.price_to_amount.get_mut_or_default(&price) += supply_state.value; + *self.price_to_amount.puts_entry_or_default(&price) += supply_state.value; } } @@ -62,7 +56,7 @@ impl CohortState { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.decrement(supply_state, price); - *self.price_to_amount.get_mut_or_default(&price) -= supply_state.value; + *self.price_to_amount.puts_entry_or_default(&price) -= supply_state.value; } } @@ -71,7 +65,7 @@ impl CohortState { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.receive(supply_state, price); - *self.price_to_amount.get_mut_or_default(&price) += supply_state.value; + *self.price_to_amount.puts_entry_or_default(&price) += supply_state.value; } } @@ -87,7 +81,7 @@ impl CohortState { let current_price = current_price.unwrap(); let prev_price = prev_price.unwrap(); realized.send(supply_state, current_price, prev_price, older_than_hour); - *self.price_to_amount.get_mut_or_default(&prev_price) -= supply_state.value; + *self.price_to_amount.puts_entry_or_default(&prev_price) -= supply_state.value; } } @@ -114,21 +108,32 @@ impl CohortState { } Ordering::Less => { state.supply_in_profit += sats; - // if price > Dollars::ZERO { - // state.unrealized_profit += - // current_price.checked_sub(price).unwrap() * sats; - // } + if price > Dollars::ZERO && current_price > Dollars::ZERO { + let diff = current_price.checked_sub(price).unwrap(); + if diff <= Dollars::ZERO { + dbg!(price, current_price, diff, sats); + panic!(); + } + state.unrealized_profit += diff * sats; + } } Ordering::Greater => { state.supply_in_loss += sats; - // state.unrealized_loss += price.checked_sub(current_price).unwrap() * sats; + if price > Dollars::ZERO && current_price > Dollars::ZERO { + let diff = price.checked_sub(current_price).unwrap(); + if diff <= Dollars::ZERO { + dbg!(price, current_price, diff, sats); + panic!(); + } + state.unrealized_loss += diff * sats; + } } } }; self.price_to_amount - .unordered_clone_iter() - .for_each(|(price, sats)| { + .puts_iter() + .for_each(|(&price, &sats)| { update_state(price, height_price, sats, &mut height_unrealized_state); if let Some(date_price) = date_price { @@ -143,4 +148,13 @@ impl CohortState { (height_unrealized_state, date_unrealized_state) } + + pub fn commit(&mut self, height: Height) -> Result<()> { + self.price_to_amount + .retain_or_del(|_, sats| *sats > Sats::ZERO); + let price_to_amount_puts = self.price_to_amount.clone_puts(); + self.price_to_amount.commit(height)?; + self.price_to_amount.append_puts(price_to_amount_puts); + Ok(()) + } } diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index 7440d1f1f..7b36e8934 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -9,8 +9,7 @@ license.workspace = true repository.workspace = true [dependencies] +arc-swap = { workspace = true } brk_core = { workspace = true } byteview = { workspace = true } -color-eyre = { workspace = true } fjall = { workspace = true } -zerocopy = { workspace = true } diff --git a/crates/brk_store/README.md b/crates/brk_store/README.md new file mode 100644 index 000000000..3e64ce0d1 --- /dev/null +++ b/crates/brk_store/README.md @@ -0,0 +1 @@ +# BRK Store diff --git a/crates/brk_store/examples/main.rs b/crates/brk_store/examples/main.rs new file mode 100644 index 000000000..e8fced8a4 --- /dev/null +++ b/crates/brk_store/examples/main.rs @@ -0,0 +1,43 @@ +use std::path::Path; + +use brk_core::{Dollars, Height, Result, Sats, Version}; +use brk_store::Store; + +fn main() -> Result<()> { + let p = Path::new("./examples/_fjall"); + + let keyspace = brk_store::open_keyspace(p)?; + + let mut store: Store = + brk_store::Store::import(&keyspace, p, "n", Version::ZERO, None)?; + + store.copy_db_to_puts(); + + *store.puts_entry_or_default(&Dollars::from(10.0)) += Sats::ONE_BTC; + *store.puts_entry_or_default(&Dollars::from(1.0)) += Sats::ONE_BTC; + *store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC; + *store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC; + + dbg!(store.tx_iter().collect::>()); + + store.commit(Height::ZERO)?; + + store.copy_db_to_puts(); + + dbg!(store.tx_iter().collect::>()); + + *store.puts_entry_or_default(&Dollars::from(10.0)) += Sats::ONE_BTC; + *store.puts_entry_or_default(&Dollars::from(1.0)) += Sats::ONE_BTC; + *store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC; + *store.puts_entry_or_default(&Dollars::ZERO) += Sats::ONE_BTC; + + dbg!(store.tx_iter().collect::>()); + + store.commit(Height::from(1_u32))?; + + store.copy_db_to_puts(); + + dbg!(store.tx_iter().collect::>()); + + Ok(()) +} diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 9be1ebe7e..175ae226f 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -1,18 +1,23 @@ +#![doc = include_str!("../README.md")] +#![doc = "\n## Example\n\n```rust"] +#![doc = include_str!("../examples/main.rs")] +#![doc = "```"] + use std::{ collections::{BTreeMap, BTreeSet}, - error, fmt::Debug, mem, path::Path, + sync::Arc, }; +use arc_swap::ArcSwap; use brk_core::{Height, Result, Value, Version}; use byteview::ByteView; use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace, TransactionalPartitionHandle, }; -use zerocopy::{Immutable, IntoBytes}; mod meta; use meta::*; @@ -21,7 +26,7 @@ pub struct Store { meta: StoreMeta, name: String, keyspace: TransactionalKeyspace, - partition: TransactionalPartitionHandle, + partition: Arc>, rtx: ReadTransaction, puts: BTreeMap, dels: BTreeSet, @@ -30,15 +35,20 @@ pub struct Store { /// Use default if will read const DEFAULT_BLOOM_FILTER_BITS: Option = Some(5); -const CHECK_COLLISISONS: bool = true; +// const CHECK_COLLISISONS: bool = true; const MAJOR_FJALL_VERSION: Version = Version::TWO; -impl Store +pub fn open_keyspace(path: &Path) -> fjall::Result { + fjall::Config::new(path.join("fjall")) + .max_write_buffer_size(32 * 1024 * 1024) + .open_transactional() +} + +impl<'a, K, V> Store where - K: Debug + Clone + Into + TryFrom + Ord + Immutable + IntoBytes, - V: Debug + Clone + Into + TryFrom, - >::Error: error::Error + Send + Sync + 'static, - >::Error: error::Error + Send + Sync + 'static, + K: Debug + Clone + From + Ord + 'a, + V: Debug + Clone + From, + ByteView: From + From<&'a K> + From, { pub fn import( keyspace: &TransactionalKeyspace, @@ -65,7 +75,7 @@ where meta, name: name.to_owned(), keyspace: keyspace.clone(), - partition, + partition: Arc::new(ArcSwap::from_pointee(partition)), rtx, puts: BTreeMap::new(), dels: BTreeSet::new(), @@ -73,39 +83,70 @@ where }) } - pub fn get(&self, key: &K) -> color_eyre::Result>> { + pub fn get(&self, key: &'a K) -> Result>> { if let Some(v) = self.puts.get(key) { Ok(Some(Value::Ref(v))) - } else if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes())? { - Ok(Some(Value::Owned(V::try_from(slice.as_bytes().into())?))) + } else if let Some(slice) = self.rtx.get(&self.partition.load(), ByteView::from(key))? { + Ok(Some(Value::Owned(V::from(ByteView::from(slice))))) } else { Ok(None) } } - pub fn get_mut_or_default(&mut self, key: &K) -> &mut V + pub fn first_key_value(&self) -> Result> { + Ok(self + .rtx + .first_key_value(&self.partition.load())? + .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v))))) + } + + pub fn last_key_value(&self) -> Result> { + Ok(self + .rtx + .last_key_value(&self.partition.load())? + .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v))))) + } + + pub fn puts_entry_or_default(&mut self, key: &'a K) -> &mut V where V: Default, { - self.puts.entry(key.clone()).or_insert_with(|| { - if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes()).unwrap() { - V::try_from(slice.as_bytes().into()).unwrap() - } else { - V::default() - } - }) + self.puts.entry(key.clone()).or_default() } - pub fn unordered_clone_iter(&self) -> impl Iterator { + pub fn tx_iter(&self) -> impl Iterator { self.rtx - .iter(&self.partition) + .iter(&self.partition.load()) .map(|res| res.unwrap()) - .map(|(k, v)| (K::try_from(ByteView::from(k)).unwrap(), v)) - .filter(|(k, _)| !self.puts.contains_key(k) && !self.dels.contains(k)) - .map(|(k, v)| (k, V::try_from(ByteView::from(v)).unwrap())) - .chain(self.puts.iter().map(|(k, v)| (k.clone(), v.clone()))) + .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))) } + pub fn puts_iter(&self) -> impl Iterator { + self.puts.iter() + } + + pub fn clone_puts(&self) -> BTreeMap { + self.puts.clone() + } + + pub fn append_puts(&mut self, mut other: BTreeMap) { + self.puts.append(&mut other); + } + + pub fn copy_db_to_puts(&mut self) { + self.append_puts(self.tx_iter().collect()); + } + + // pub fn unordered_clone_iter(&self) -> impl Iterator { + // self.rtx + // .keys(&self.partition.load()) + // .map(|res| res.unwrap()) + // .map(|k| K::from(ByteView::from(k))) + // .filter(|k| !self.puts.contains_key(k) && !self.dels.contains(k)) + // .map(|k| (k, self.rtx.get(partition, key) V::from(ByteView::from(v)))) + // .chain(self.puts.iter().map(|(k, v)| (k.clone(), v.clone()))) + // } + pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) { if self.needs(height) { if !self.dels.is_empty() { @@ -153,30 +194,28 @@ where let mut wtx = self.keyspace.write_tx(); + let partition = &self.partition.load(); + mem::take(&mut self.dels) .into_iter() - .for_each(|key| wtx.remove(&self.partition, key.as_bytes())); + .for_each(|key| wtx.remove(partition, ByteView::from(key))); mem::take(&mut self.puts) .into_iter() .for_each(|(key, value)| { - if CHECK_COLLISISONS { - #[allow(unused_must_use)] - if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) { - dbg!( - &key, - V::try_from(value.as_bytes().into()).unwrap(), - &self.meta, - self.rtx.get(&self.partition, key.as_bytes()) - ); - unreachable!(); - } - } - wtx.insert( - &self.partition, - key.as_bytes(), - &*ByteView::try_from(value).unwrap(), - ) + // if CHECK_COLLISISONS { + // #[allow(unused_must_use)] + // if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) { + // dbg!( + // &key, + // V::try_from(value.as_bytes().into()).unwrap(), + // &self.meta, + // self.rtx.get(&self.partition, key.as_bytes()) + // ); + // unreachable!(); + // } + // } + wtx.insert(partition, ByteView::from(key), ByteView::from(value)) }); wtx.commit()?; @@ -187,7 +226,7 @@ where } pub fn rotate_memtable(&self) { - let _ = self.partition.inner().rotate_memtable(); + let _ = self.partition.load().inner().rotate_memtable(); } pub fn height(&self) -> Option { @@ -225,10 +264,23 @@ where } pub fn reset_partition(&mut self) -> Result<()> { - self.keyspace.delete_partition(self.partition.clone())?; + let partition = Arc::try_unwrap(self.partition.swap(unsafe { + #[allow(clippy::uninit_assumed_init, invalid_value)] + mem::MaybeUninit::uninit().assume_init() + })) + .ok() + .unwrap(); + + self.keyspace.delete_partition(partition)?; + self.keyspace.persist(PersistMode::SyncAll)?; - self.partition = - Self::open_partition_handle(&self.keyspace, &self.name, self.bloom_filter_bits)?; + + self.partition.store(Arc::new(Self::open_partition_handle( + &self.keyspace, + &self.name, + self.bloom_filter_bits, + )?)); + Ok(()) } } diff --git a/crates/brk_store/src/meta.rs b/crates/brk_store/src/meta.rs index 71f3b5e42..c763f2c46 100644 --- a/crates/brk_store/src/meta.rs +++ b/crates/brk_store/src/meta.rs @@ -3,9 +3,8 @@ use std::{ path::{Path, PathBuf}, }; -use brk_core::{Result, Version}; +use brk_core::{Result, Version, copy_first_8bytes}; use fjall::{TransactionalKeyspace, TransactionalPartitionHandle}; -use zerocopy::{FromBytes, IntoBytes}; use super::Height; @@ -48,7 +47,7 @@ impl StoreMeta { pathbuf: path.to_owned(), version, height: Height::try_from(Self::path_height_(path).as_path()).ok(), - len: Self::read_length_(path)?, + len: Self::read_length_(path), }; slf.version.write(&slf.path_version())?; @@ -109,16 +108,16 @@ impl StoreMeta { path.join("height") } - fn read_length_(path: &Path) -> Result { - Ok(fs::read(Self::path_length(path)) - .map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default()) - .unwrap_or_default()) + fn read_length_(path: &Path) -> usize { + fs::read(Self::path_length(path)) + .map(|v| usize::from_ne_bytes(copy_first_8bytes(v.as_slice()).unwrap_or_default())) + .unwrap_or_default() } fn write_length(&self) -> io::Result<()> { Self::write_length_(&self.pathbuf, self.len) } fn write_length_(path: &Path, len: usize) -> io::Result<()> { - fs::write(Self::path_length(path), len.as_bytes()) + fs::write(Self::path_length(path), len.to_ne_bytes()) } fn path_length(path: &Path) -> PathBuf { path.join("length") diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index a4feaa850..2b9a664db 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true repository.workspace = true [dependencies] -arc-swap = "1.7.1" +arc-swap = { workspace = true } brk_core = { workspace = true } brk_exit = { workspace = true } clap = { workspace = true } @@ -22,3 +22,6 @@ serde_json = { workspace = true } zerocopy = { workspace = true } zerocopy-derive = { workspace = true } zstd = "0.13.3" + +[package.metadata.cargo-machete] +ignored = ["clap"] diff --git a/websites/kibo.money/scripts/main.js b/websites/kibo.money/scripts/main.js index 0e9f7601f..a556a79e3 100644 --- a/websites/kibo.money/scripts/main.js +++ b/websites/kibo.money/scripts/main.js @@ -702,6 +702,9 @@ function createUtils() { (!unit || thoroughUnitCheck) && (id.includes("in-sats") || id.endsWith("supply") || + id.endsWith("supply-even") || + id.endsWith("supply-in-profit") || + id.endsWith("supply-in-loss") || id.endsWith("stack") || (id.endsWith("value") && !id.includes("realized")) || ((id.includes("coinbase") || diff --git a/websites/kibo.money/scripts/options.js b/websites/kibo.money/scripts/options.js index 0ef222c2b..65ecc4dd1 100644 --- a/websites/kibo.money/scripts/options.js +++ b/websites/kibo.money/scripts/options.js @@ -1154,8 +1154,27 @@ function createPartialOptions(colors) { createBaseSeries({ key: `${key}supply`, name: useGroupName ? name : "Supply", - color: color, + color: "list" in args ? color : colors.default, }), + ...(!("list" in args) + ? [ + createBaseSeries({ + key: `${key}supply-in-profit`, + name: useGroupName ? name : "In Profit", + color: colors.green, + }), + createBaseSeries({ + key: `${key}supply-in-loss`, + name: useGroupName ? name : "In Loss", + color: colors.red, + }), + createBaseSeries({ + key: `${key}supply-even`, + name: useGroupName ? name : "Even", + color: colors.yellow, + }), + ] + : []), createBaseSeries({ key: `${key}supply-in-btc`, name: useGroupName ? name : "Supply", @@ -1346,6 +1365,39 @@ function createPartialOptions(colors) { }, ], }, + { + name: "Unrealized", + tree: [ + { + name: "profit", + title: `${args.title} Unrealized Profit`, + bottom: list.flatMap(({ color, name, key: _key }) => { + const key = fixKey(_key); + return /** @type {const} */ ([ + createBaseSeries({ + key: `${key}unrealized-profit`, + name: useGroupName ? name : "Profit", + color: useGroupName ? color : colors.green, + }), + ]); + }), + }, + { + name: "loss", + title: `${args.title} Unrealized Loss`, + bottom: list.flatMap(({ color, name, key: _key }) => { + const key = fixKey(_key); + return /** @type {const} */ ([ + createBaseSeries({ + key: `${key}unrealized-loss`, + name: useGroupName ? name : "Loss", + color: useGroupName ? color : colors.red, + }), + ]); + }), + }, + ], + }, ], }); }