diff --git a/Cargo.lock b/Cargo.lock index c2262932e..f480008ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -590,7 +590,7 @@ dependencies = [ "brk_vec", "color-eyre", "derive_deref", - "schemars 1.0.1", + "schemars 1.0.3", "serde", "serde_json", "serde_with", @@ -650,7 +650,7 @@ dependencies = [ "paste", "pin-project-lite", "rand 0.9.1", - "schemars 1.0.1", + "schemars 1.0.3", "serde", "serde_json", "sse-stream", @@ -2387,9 +2387,9 @@ dependencies = [ [[package]] name = "minreq" -version = "2.13.4" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d2aaba477837b46ec1289588180fabfccf0c3b1d1a0c6b1866240cd6cd5ce9" +checksum = "84885312a86831bff4a3cb04a1e54a3f698407e3274c83249313f194d3e0b678" dependencies = [ "log", "rustls", @@ -3564,9 +3564,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe8c9d1c68d67dd9f97ecbc6f932b60eb289c5dbddd8aa1405484a8fd2fcd984" +checksum = "1375ba8ef45a6f15d83fa8748f1079428295d403d6ea991d09ab100155fbc06d" dependencies = [ "dyn-clone", "ref-cast", @@ -3577,9 +3577,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ca9fcb757952f8e8629b9ab066fc62da523c46c2b247b1708a3be06dd82530b" +checksum = "2b13ed22d6d49fe23712e068770b5c4df4a693a2b02eeff8e7ca3135627a24f6" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index cdc0ec231..b3af8e2a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,9 +47,9 @@ derive_deref = "1.1.1" fjall = "2.11.1" jiff = "0.2.15" log = { version = "0.4.27" } -minreq = { version = "2.13.4", features = ["https", "serde_json"] } +minreq = { version = "2.14.0", features = ["https", "serde_json"] } rayon = "1.10.0" -schemars = "1.0.1" +schemars = "1.0.3" serde = { version = "1.0.219" } serde_bytes = "0.11.17" serde_derive = "1.0.219" diff --git a/crates/brk_bundler/src/lib.rs b/crates/brk_bundler/src/lib.rs index 7da9996ef..c86a16872 100644 --- a/crates/brk_bundler/src/lib.rs +++ b/crates/brk_bundler/src/lib.rs @@ -81,7 +81,7 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i let mut entry_watcher = notify::recommended_watcher( move |res: Result| match res { Ok(_) => write_index_clone(), - Err(e) => error!("watch error: {:?}", e), + Err(e) => error!("watch error: {e:?}"), }, ) .unwrap(); @@ -112,7 +112,7 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i let _ = fs::copy(&source_path, &dist_path); } }), - Err(e) => error!("watch error: {:?}", e), + Err(e) => error!("watch error: {e:?}"), }, ) .unwrap(); diff --git a/crates/brk_computer/src/stores.rs b/crates/brk_computer/src/stores.rs index f507b1b1a..08783268c 100644 --- a/crates/brk_computer/src/stores.rs +++ b/crates/brk_computer/src/stores.rs @@ -1,12 +1,13 @@ use std::{path::Path, thread}; use brk_core::{ - AddressData, EmptyAddressData, Height, P2AAddressIndex, P2AAddressIndexOutputindex, - P2PK33AddressIndex, P2PK33AddressIndexOutputindex, P2PK65AddressIndex, - P2PK65AddressIndexOutputindex, P2PKHAddressIndex, P2PKHAddressIndexOutputindex, - P2SHAddressIndex, P2SHAddressIndexOutputindex, P2TRAddressIndex, P2TRAddressIndexOutputindex, - P2WPKHAddressIndex, P2WPKHAddressIndexOutputindex, P2WSHAddressIndex, - P2WSHAddressIndexOutputindex, Result, Unit, Version, + AddressData, AddressIndexToTypeIndedToOutputIndex, EmptyAddressData, GroupedByAddressType, + Height, P2AAddressIndex, P2AAddressIndexOutputindex, P2PK33AddressIndex, + P2PK33AddressIndexOutputindex, P2PK65AddressIndex, P2PK65AddressIndexOutputindex, + P2PKHAddressIndex, P2PKHAddressIndexOutputindex, P2SHAddressIndex, P2SHAddressIndexOutputindex, + P2TRAddressIndex, P2TRAddressIndexOutputindex, P2WPKHAddressIndex, + P2WPKHAddressIndexOutputindex, P2WSHAddressIndex, P2WSHAddressIndexOutputindex, Result, Unit, + Version, }; use brk_store::{AnyStore, Store}; use fjall::{PersistMode, TransactionalKeyspace}; @@ -20,43 +21,43 @@ pub struct Stores { pub p2aaddressindex_to_addressdata: Store, pub p2aaddressindex_to_emptyaddressdata: Store, - pub p2aaddressindex_to_utxos_received: Store, - pub p2aaddressindex_to_utxos_sent: Store, + pub p2aaddressindex_to_outputs_received: Store, + pub p2aaddressindex_to_outputs_sent: Store, pub p2pk33addressindex_to_addressdata: Store, pub p2pk33addressindex_to_emptyaddressdata: Store, - pub p2pk33addressindex_to_utxos_received: Store, - pub p2pk33addressindex_to_utxos_sent: Store, + pub p2pk33addressindex_to_outputs_received: Store, + pub p2pk33addressindex_to_outputs_sent: Store, pub p2pk65addressindex_to_addressdata: Store, pub p2pk65addressindex_to_emptyaddressdata: Store, - pub p2pk65addressindex_to_utxos_received: Store, - pub p2pk65addressindex_to_utxos_sent: Store, + pub p2pk65addressindex_to_outputs_received: Store, + pub p2pk65addressindex_to_outputs_sent: Store, pub p2pkhaddressindex_to_addressdata: Store, pub p2pkhaddressindex_to_emptyaddressdata: Store, - pub p2pkhaddressindex_to_utxos_received: Store, - pub p2pkhaddressindex_to_utxos_sent: Store, + pub p2pkhaddressindex_to_outputs_received: Store, + pub p2pkhaddressindex_to_outputs_sent: Store, pub p2shaddressindex_to_addressdata: Store, pub p2shaddressindex_to_emptyaddressdata: Store, - pub p2shaddressindex_to_utxos_received: Store, - pub p2shaddressindex_to_utxos_sent: Store, + pub p2shaddressindex_to_outputs_received: Store, + pub p2shaddressindex_to_outputs_sent: Store, pub p2traddressindex_to_addressdata: Store, pub p2traddressindex_to_emptyaddressdata: Store, - pub p2traddressindex_to_utxos_received: Store, - pub p2traddressindex_to_utxos_sent: Store, + pub p2traddressindex_to_outputs_received: Store, + pub p2traddressindex_to_outputs_sent: Store, pub p2wpkhaddressindex_to_addressdata: Store, pub p2wpkhaddressindex_to_emptyaddressdata: Store, - pub p2wpkhaddressindex_to_utxos_received: Store, - pub p2wpkhaddressindex_to_utxos_sent: Store, + pub p2wpkhaddressindex_to_outputs_received: Store, + pub p2wpkhaddressindex_to_outputs_sent: Store, pub p2wshaddressindex_to_addressdata: Store, pub p2wshaddressindex_to_emptyaddressdata: Store, - pub p2wshaddressindex_to_utxos_received: Store, - pub p2wshaddressindex_to_utxos_sent: Store, + pub p2wshaddressindex_to_outputs_received: Store, + pub p2wshaddressindex_to_outputs_sent: Store, } impl Stores { @@ -69,50 +70,50 @@ impl Stores { ( p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata, - p2aaddressindex_to_utxos_received, - p2aaddressindex_to_utxos_sent, + p2aaddressindex_to_outputs_received, + p2aaddressindex_to_outputs_sent, ), ( p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata, - p2pk33addressindex_to_utxos_received, - p2pk33addressindex_to_utxos_sent, + p2pk33addressindex_to_outputs_received, + p2pk33addressindex_to_outputs_sent, ), ( p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata, - p2pk65addressindex_to_utxos_received, - p2pk65addressindex_to_utxos_sent, + p2pk65addressindex_to_outputs_received, + p2pk65addressindex_to_outputs_sent, ), ( p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata, - p2pkhaddressindex_to_utxos_received, - p2pkhaddressindex_to_utxos_sent, + p2pkhaddressindex_to_outputs_received, + p2pkhaddressindex_to_outputs_sent, ), ( p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata, - p2shaddressindex_to_utxos_received, - p2shaddressindex_to_utxos_sent, + p2shaddressindex_to_outputs_received, + p2shaddressindex_to_outputs_sent, ), ( p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata, - p2traddressindex_to_utxos_received, - p2traddressindex_to_utxos_sent, + p2traddressindex_to_outputs_received, + p2traddressindex_to_outputs_sent, ), ( p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata, - p2wpkhaddressindex_to_utxos_received, - p2wpkhaddressindex_to_utxos_sent, + p2wpkhaddressindex_to_outputs_received, + p2wpkhaddressindex_to_outputs_sent, ), ( p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata, - p2wshaddressindex_to_utxos_received, - p2wshaddressindex_to_utxos_sent, + p2wshaddressindex_to_outputs_received, + p2wshaddressindex_to_outputs_sent, ), ) = thread::scope(|scope| { let p2a = scope.spawn(|| { @@ -136,7 +137,7 @@ impl Stores { Store::import( keyspace, path, - "p2aaddressindex_to_utxos_received", + "p2aaddressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -144,7 +145,7 @@ impl Stores { Store::import( keyspace, path, - "p2aaddressindex_to_utxos_sent", + "p2aaddressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -173,7 +174,7 @@ impl Stores { Store::import( keyspace, path, - "p2pk33addressindex_to_utxos_received", + "p2pk33addressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -181,7 +182,7 @@ impl Stores { Store::import( keyspace, path, - "p2pk33addressindex_to_utxos_sent", + "p2pk33addressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -210,7 +211,7 @@ impl Stores { Store::import( keyspace, path, - "p2pk65addressindex_to_utxos_received", + "p2pk65addressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -218,7 +219,7 @@ impl Stores { Store::import( keyspace, path, - "p2pk65addressindex_to_utxos_sent", + "p2pk65addressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -247,7 +248,7 @@ impl Stores { Store::import( keyspace, path, - "p2pkhaddressindex_to_utxos_received", + "p2pkhaddressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -255,7 +256,7 @@ impl Stores { Store::import( keyspace, path, - "p2pkhaddressindex_to_utxos_sent", + "p2pkhaddressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -284,7 +285,7 @@ impl Stores { Store::import( keyspace, path, - "p2shaddressindex_to_utxos_received", + "p2shaddressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -292,7 +293,7 @@ impl Stores { Store::import( keyspace, path, - "p2shaddressindex_to_utxos_sent", + "p2shaddressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -321,7 +322,7 @@ impl Stores { Store::import( keyspace, path, - "p2traddressindex_to_utxos_received", + "p2traddressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -329,7 +330,7 @@ impl Stores { Store::import( keyspace, path, - "p2traddressindex_to_utxos_sent", + "p2traddressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -358,7 +359,7 @@ impl Stores { Store::import( keyspace, path, - "p2wpkhaddressindex_to_utxos_received", + "p2wpkhaddressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -366,7 +367,7 @@ impl Stores { Store::import( keyspace, path, - "p2wpkhaddressindex_to_utxos_sent", + "p2wpkhaddressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -395,7 +396,7 @@ impl Stores { Store::import( keyspace, path, - "p2wshaddressindex_to_utxos_received", + "p2wshaddressindex_to_outputs_received", version + VERSION + Version::ZERO, None, ) @@ -403,7 +404,7 @@ impl Stores { Store::import( keyspace, path, - "p2wshaddressindex_to_utxos_sent", + "p2wshaddressindex_to_outputs_sent", version + VERSION + Version::ZERO, None, ) @@ -428,43 +429,43 @@ impl Stores { p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata, - p2aaddressindex_to_utxos_received, - p2aaddressindex_to_utxos_sent, + p2aaddressindex_to_outputs_received, + p2aaddressindex_to_outputs_sent, p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata, - p2pk33addressindex_to_utxos_received, - p2pk33addressindex_to_utxos_sent, + p2pk33addressindex_to_outputs_received, + p2pk33addressindex_to_outputs_sent, p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata, - p2pk65addressindex_to_utxos_received, - p2pk65addressindex_to_utxos_sent, + p2pk65addressindex_to_outputs_received, + p2pk65addressindex_to_outputs_sent, p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata, - p2pkhaddressindex_to_utxos_received, - p2pkhaddressindex_to_utxos_sent, + p2pkhaddressindex_to_outputs_received, + p2pkhaddressindex_to_outputs_sent, p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata, - p2shaddressindex_to_utxos_received, - p2shaddressindex_to_utxos_sent, + p2shaddressindex_to_outputs_received, + p2shaddressindex_to_outputs_sent, p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata, - p2traddressindex_to_utxos_received, - p2traddressindex_to_utxos_sent, + p2traddressindex_to_outputs_received, + p2traddressindex_to_outputs_sent, p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata, - p2wpkhaddressindex_to_utxos_received, - p2wpkhaddressindex_to_utxos_sent, + p2wpkhaddressindex_to_outputs_received, + p2wpkhaddressindex_to_outputs_sent, p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata, - p2wshaddressindex_to_utxos_received, - p2wshaddressindex_to_utxos_sent, + p2wshaddressindex_to_outputs_received, + p2wshaddressindex_to_outputs_sent, }) } @@ -476,10 +477,221 @@ impl Stores { .unwrap() } - pub fn commit(&mut self, height: Height) -> Result<()> { + pub fn reset(&mut self) -> Result<()> { self.as_mut_slice() .into_par_iter() - .try_for_each(|store| store.commit(height))?; + .try_for_each(|store| store.reset())?; + + self.keyspace + .persist(PersistMode::SyncAll) + .map_err(|e| e.into()) + } + + pub fn commit( + &mut self, + height: Height, + sent: AddressIndexToTypeIndedToOutputIndex, + received: AddressIndexToTypeIndedToOutputIndex, + ) -> Result<()> { + // &mut self.p2aaddressindex_to_addressdata, + // &mut self.p2pk33addressindex_to_addressdata, + // &mut self.p2pk65addressindex_to_addressdata, + // &mut self.p2pkhaddressindex_to_addressdata, + // &mut self.p2shaddressindex_to_addressdata, + // &mut self.p2traddressindex_to_addressdata, + // &mut self.p2wpkhaddressindex_to_addressdata, + // &mut self.p2wshaddressindex_to_addressdata, + + // &mut self.p2aaddressindex_to_emptyaddressdata, + // &mut self.p2pk33addressindex_to_emptyaddressdata, + // &mut self.p2pk65addressindex_to_emptyaddressdata, + // &mut self.p2pkhaddressindex_to_emptyaddressdata, + // &mut self.p2shaddressindex_to_emptyaddressdata, + // &mut self.p2traddressindex_to_emptyaddressdata, + // &mut self.p2wpkhaddressindex_to_emptyaddressdata, + // &mut self.p2wshaddressindex_to_emptyaddressdata, + + thread::scope(|s| { + let GroupedByAddressType { + p2pk65, + p2pk33, + p2pkh, + p2sh, + p2wpkh, + p2wsh, + p2tr, + p2a, + } = received.inner(); + + s.spawn(|| { + self.p2aaddressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2a.into_iter() + .map(P2AAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2pk33addressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2pk33 + .into_iter() + .map(P2PK33AddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2pk65addressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2pk65 + .into_iter() + .map(P2PK65AddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2pkhaddressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2pkh + .into_iter() + .map(P2PKHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2shaddressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2sh.into_iter() + .map(P2SHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2traddressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2tr.into_iter() + .map(P2TRAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2wpkhaddressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2wpkh + .into_iter() + .map(P2WPKHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2wshaddressindex_to_outputs_received.commit_( + height, + [].into_iter(), + p2wsh + .into_iter() + .map(P2WSHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + }); + + thread::scope(|s| { + let GroupedByAddressType { + p2pk65, + p2pk33, + p2pkh, + p2sh, + p2wpkh, + p2wsh, + p2tr, + p2a, + } = sent.inner(); + + s.spawn(|| { + self.p2aaddressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2a.into_iter() + .map(P2AAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2pk33addressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2pk33 + .into_iter() + .map(P2PK33AddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2pk65addressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2pk65 + .into_iter() + .map(P2PK65AddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2pkhaddressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2pkh + .into_iter() + .map(P2PKHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2shaddressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2sh.into_iter() + .map(P2SHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2traddressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2tr.into_iter() + .map(P2TRAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2wpkhaddressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2wpkh + .into_iter() + .map(P2WPKHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + s.spawn(|| { + self.p2wshaddressindex_to_outputs_sent.commit_( + height, + [].into_iter(), + p2wsh + .into_iter() + .map(P2WSHAddressIndexOutputindex::from) + .map(|i| (i, Unit)), + ) + }); + }); self.keyspace .persist(PersistMode::SyncAll) @@ -492,40 +704,40 @@ impl Stores { .for_each(|store| store.rotate_memtable()); } - fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 32] { + pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 32] { [ &self.p2aaddressindex_to_addressdata, &self.p2aaddressindex_to_emptyaddressdata, - &self.p2aaddressindex_to_utxos_received, - &self.p2aaddressindex_to_utxos_sent, + &self.p2aaddressindex_to_outputs_received, + &self.p2aaddressindex_to_outputs_sent, &self.p2pk33addressindex_to_addressdata, &self.p2pk33addressindex_to_emptyaddressdata, - &self.p2pk33addressindex_to_utxos_received, - &self.p2pk33addressindex_to_utxos_sent, + &self.p2pk33addressindex_to_outputs_received, + &self.p2pk33addressindex_to_outputs_sent, &self.p2pk65addressindex_to_addressdata, &self.p2pk65addressindex_to_emptyaddressdata, - &self.p2pk65addressindex_to_utxos_received, - &self.p2pk65addressindex_to_utxos_sent, + &self.p2pk65addressindex_to_outputs_received, + &self.p2pk65addressindex_to_outputs_sent, &self.p2pkhaddressindex_to_addressdata, &self.p2pkhaddressindex_to_emptyaddressdata, - &self.p2pkhaddressindex_to_utxos_received, - &self.p2pkhaddressindex_to_utxos_sent, + &self.p2pkhaddressindex_to_outputs_received, + &self.p2pkhaddressindex_to_outputs_sent, &self.p2shaddressindex_to_addressdata, &self.p2shaddressindex_to_emptyaddressdata, - &self.p2shaddressindex_to_utxos_received, - &self.p2shaddressindex_to_utxos_sent, + &self.p2shaddressindex_to_outputs_received, + &self.p2shaddressindex_to_outputs_sent, &self.p2traddressindex_to_addressdata, &self.p2traddressindex_to_emptyaddressdata, - &self.p2traddressindex_to_utxos_received, - &self.p2traddressindex_to_utxos_sent, + &self.p2traddressindex_to_outputs_received, + &self.p2traddressindex_to_outputs_sent, &self.p2wpkhaddressindex_to_addressdata, &self.p2wpkhaddressindex_to_emptyaddressdata, - &self.p2wpkhaddressindex_to_utxos_received, - &self.p2wpkhaddressindex_to_utxos_sent, + &self.p2wpkhaddressindex_to_outputs_received, + &self.p2wpkhaddressindex_to_outputs_sent, &self.p2wshaddressindex_to_addressdata, &self.p2wshaddressindex_to_emptyaddressdata, - &self.p2wshaddressindex_to_utxos_received, - &self.p2wshaddressindex_to_utxos_sent, + &self.p2wshaddressindex_to_outputs_received, + &self.p2wshaddressindex_to_outputs_sent, ] } @@ -533,36 +745,36 @@ impl Stores { [ &mut self.p2aaddressindex_to_addressdata, &mut self.p2aaddressindex_to_emptyaddressdata, - &mut self.p2aaddressindex_to_utxos_received, - &mut self.p2aaddressindex_to_utxos_sent, + &mut self.p2aaddressindex_to_outputs_received, + &mut self.p2aaddressindex_to_outputs_sent, &mut self.p2pk33addressindex_to_addressdata, &mut self.p2pk33addressindex_to_emptyaddressdata, - &mut self.p2pk33addressindex_to_utxos_received, - &mut self.p2pk33addressindex_to_utxos_sent, + &mut self.p2pk33addressindex_to_outputs_received, + &mut self.p2pk33addressindex_to_outputs_sent, &mut self.p2pk65addressindex_to_addressdata, &mut self.p2pk65addressindex_to_emptyaddressdata, - &mut self.p2pk65addressindex_to_utxos_received, - &mut self.p2pk65addressindex_to_utxos_sent, + &mut self.p2pk65addressindex_to_outputs_received, + &mut self.p2pk65addressindex_to_outputs_sent, &mut self.p2pkhaddressindex_to_addressdata, &mut self.p2pkhaddressindex_to_emptyaddressdata, - &mut self.p2pkhaddressindex_to_utxos_received, - &mut self.p2pkhaddressindex_to_utxos_sent, + &mut self.p2pkhaddressindex_to_outputs_received, + &mut self.p2pkhaddressindex_to_outputs_sent, &mut self.p2shaddressindex_to_addressdata, &mut self.p2shaddressindex_to_emptyaddressdata, - &mut self.p2shaddressindex_to_utxos_received, - &mut self.p2shaddressindex_to_utxos_sent, + &mut self.p2shaddressindex_to_outputs_received, + &mut self.p2shaddressindex_to_outputs_sent, &mut self.p2traddressindex_to_addressdata, &mut self.p2traddressindex_to_emptyaddressdata, - &mut self.p2traddressindex_to_utxos_received, - &mut self.p2traddressindex_to_utxos_sent, + &mut self.p2traddressindex_to_outputs_received, + &mut self.p2traddressindex_to_outputs_sent, &mut self.p2wpkhaddressindex_to_addressdata, &mut self.p2wpkhaddressindex_to_emptyaddressdata, - &mut self.p2wpkhaddressindex_to_utxos_received, - &mut self.p2wpkhaddressindex_to_utxos_sent, + &mut self.p2wpkhaddressindex_to_outputs_received, + &mut self.p2wpkhaddressindex_to_outputs_sent, &mut self.p2wshaddressindex_to_addressdata, &mut self.p2wshaddressindex_to_emptyaddressdata, - &mut self.p2wshaddressindex_to_utxos_received, - &mut self.p2wshaddressindex_to_utxos_sent, + &mut self.p2wshaddressindex_to_outputs_received, + &mut self.p2wshaddressindex_to_outputs_sent, ] } } diff --git a/crates/brk_computer/src/vecs/stateful/mod.rs b/crates/brk_computer/src/vecs/stateful/mod.rs index 80a3e0dbd..24a2bfb67 100644 --- a/crates/brk_computer/src/vecs/stateful/mod.rs +++ b/crates/brk_computer/src/vecs/stateful/mod.rs @@ -1,8 +1,8 @@ -use std::{cmp::Ordering, collections::BTreeMap, path::Path, thread}; +use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread}; use brk_core::{ - DateIndex, GroupedByAddressType, Height, InputIndex, OutputIndex, OutputType, Result, Sats, - StoredUsize, Version, + AddressIndexToTypeIndedToOutputIndex, DateIndex, GroupedByAddressType, Height, InputIndex, + OutputIndex, OutputType, Result, Sats, StoredUsize, Version, }; use brk_exit::Exit; use brk_indexer::Indexer; @@ -200,6 +200,7 @@ impl Vecs { let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed; let outputindex_to_txindex = &indexes.outputindex_to_txindex; let outputindex_to_outputtype = &indexer.vecs.outputindex_to_outputtype; + let outputindex_to_typeindex = &indexer.vecs.outputindex_to_typeindex; let height_to_unclaimed_rewards = transactions .indexes_to_unclaimed_rewards .sats @@ -219,6 +220,7 @@ impl Vecs { let inputindex_to_outputindex_mmap = inputindex_to_outputindex.mmap().load(); let outputindex_to_value_mmap = outputindex_to_value.mmap().load(); let outputindex_to_outputtype_mmap = outputindex_to_outputtype.mmap().load(); + let outputindex_to_typeindex_mmap = outputindex_to_typeindex.mmap().load(); let outputindex_to_txindex_mmap = outputindex_to_txindex.mmap().load(); let txindex_to_height_mmap = txindex_to_height.mmap().load(); @@ -226,9 +228,7 @@ impl Vecs { let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); let mut height_to_output_count_iter = height_to_output_count.into_iter(); let mut height_to_input_count_iter = height_to_input_count.into_iter(); - // let mut outputindex_to_value_iter_2 = outputindex_to_value.into_iter(); let mut height_to_close_iter = height_to_close.as_ref().map(|v| v.into_iter()); - // let mut outputindex_to_outputtype_iter_2 = outputindex_to_outputtype.into_iter(); let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter(); let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter(); let mut dateindex_to_close_iter = dateindex_to_close.as_ref().map(|v| v.into_iter()); @@ -249,6 +249,7 @@ impl Vecs { + txindex_to_height.version() + outputindex_to_txindex.version() + outputindex_to_outputtype.version() + + outputindex_to_typeindex.version() + height_to_unclaimed_rewards.version() + height_to_close .as_ref() @@ -258,7 +259,8 @@ impl Vecs { .map_or(Version::ZERO, |v| v.version()) + height_to_date_fixed.version() + dateindex_to_first_height.version() - + dateindex_to_height_count.version(); + + dateindex_to_height_count.version() + + stores.as_slice().into_iter().map(|s| s.version()).sum(); separate_utxo_vecs .par_iter_mut() @@ -281,6 +283,9 @@ impl Vecs { .min() .unwrap_or_default() .min(chain_state_starting_height) + .min(stores.starting_height()) + .min(Height::from(self.height_to_unspendable_supply.len())) + .min(Height::from(self.height_to_opreturn_supply.len())) .cmp(&chain_state_starting_height) { Ordering::Greater => unreachable!(), @@ -308,11 +313,7 @@ impl Vecs { Ordering::Less => Height::ZERO, }; - let starting_height = starting_indexes - .height - .min(stateful_starting_height) - .min(Height::from(self.height_to_unspendable_supply.len())) - .min(Height::from(self.height_to_opreturn_supply.len())); + let starting_height = starting_indexes.height.min(stateful_starting_height); if starting_height.is_zero() { info!("Starting processing utxos from the start"); @@ -321,6 +322,8 @@ impl Vecs { chain_state = vec![]; chain_state_starting_height = Height::ZERO; + stores.reset()?; + separate_utxo_vecs .par_iter_mut() .try_for_each(|(_, v)| v.state.price_to_amount.reset())?; @@ -350,6 +353,11 @@ impl Vecs { let mut height = starting_height; + let mut addressindex_to_typedindex_to_sent_outputindex = + AddressIndexToTypeIndedToOutputIndex::default(); + let mut addressindex_to_typedindex_to_received_outputindex = + AddressIndexToTypeIndedToOutputIndex::default(); + (height.unwrap_to_usize()..height_to_date_fixed.len()) .map(Height::from) .try_for_each(|_height| -> color_eyre::Result<()> { @@ -375,7 +383,7 @@ impl Vecs { let output_count = height_to_output_count_iter.unwrap_get_inner(height); let input_count = height_to_input_count_iter.unwrap_get_inner(height); - let (mut height_to_sent, mut received) = thread::scope(|s| { + let ((mut height_to_sent, new_addressindex_to_typedindex_to_sent_outputindex), (mut received, new_addressindex_to_typedindex_to_received_outputindex)) = thread::scope(|s| { if chain_state_starting_height <= height { s.spawn(|| { self.utxos_vecs @@ -407,12 +415,18 @@ impl Vecs { .unwrap() .into_owned(); - // dbg!(input_type); + let typeindex = outputindex_to_typeindex + .get_or_read(outputindex, &outputindex_to_typeindex_mmap) + .unwrap() + .unwrap() + .into_owned(); if input_type.is_unspendable() { unreachable!() } + // stores. + let input_txindex = outputindex_to_txindex .get_or_read(outputindex, &outputindex_to_txindex_mmap) .unwrap() @@ -425,25 +439,39 @@ impl Vecs { .unwrap() .into_owned(); - (height, value, input_type) + (height, value, input_type, typeindex, outputindex) }) .fold( - BTreeMap::::default, - |mut tree, (height, value, input_type)| { + || { + ( + BTreeMap::::default(), + AddressIndexToTypeIndedToOutputIndex::default(), + ) + }, + |(mut tree, mut vecs), (height, value, input_type, typeindex, outputindex)| { tree.entry(height).or_default().iterate(value, input_type); - tree + if let Some( vec) = vecs.get_mut(input_type) { + vec.push((typeindex, outputindex)); + } + (tree, vecs) }, ) - .reduce(BTreeMap::::default, |first, second| { - let (mut source, to_consume) = if first.len() > second.len() { - (first, second) + .reduce( || { + ( + BTreeMap::::default(), + AddressIndexToTypeIndedToOutputIndex::default(), + ) + }, |(first_tree, mut source_vecs), (second_tree, other_vecs)| { + let (mut tree_source, tree_to_consume) = if first_tree.len() > second_tree.len() { + (first_tree, second_tree) } else { - (second, first) + (second_tree, first_tree) }; - to_consume.into_iter().for_each(|(k, v)| { - *source.entry(k).or_default() += v; + tree_to_consume.into_iter().for_each(|(k, v)| { + *tree_source.entry(k).or_default() += v; }); - source + source_vecs.merge(other_vecs); + (tree_source, source_vecs) }) }); @@ -464,21 +492,36 @@ impl Vecs { .unwrap() .into_owned(); - (value, output_type) + let typeindex = outputindex_to_typeindex + .get_or_read(outputindex, &outputindex_to_typeindex_mmap) + .unwrap() + .unwrap() + .into_owned(); + + (value, output_type, typeindex, outputindex) }) .fold( - Transacted::default, - |mut transacted, (value, output_type)| { + || (Transacted::default(), AddressIndexToTypeIndedToOutputIndex::default()), + |(mut transacted, mut vecs), (value, output_type, typeindex, outputindex)| { transacted.iterate(value, output_type); - transacted + if let Some(vec) = vecs.get_mut(output_type) { + vec.push((typeindex, outputindex)); + } + (transacted, vecs) }, ) - .reduce(Transacted::default, |acc, transacted| acc + transacted) + .reduce(|| (Transacted::default(), AddressIndexToTypeIndedToOutputIndex::default()), |(transacted, mut vecs), (other_transacted, other_vecs)| { + vecs.merge(other_vecs); + (transacted + other_transacted, vecs) + }) }); (sent_handle.join().unwrap(), received_handle.join().unwrap()) }); + addressindex_to_typedindex_to_sent_outputindex.merge(new_addressindex_to_typedindex_to_sent_outputindex); + addressindex_to_typedindex_to_received_outputindex.merge(new_addressindex_to_typedindex_to_received_outputindex); + unspendable_supply += received .by_type .unspendable @@ -564,10 +607,19 @@ impl Vecs { ) })?; - if height != Height::ZERO && height.unwrap_to_usize() % 20_000 == 0 { + if height != Height::ZERO && height.unwrap_to_usize() % 10_000 == 0 { info!("Flushing..."); exit.block(); - self.flush_states(height, &chain_state, exit)?; + self.flush_states( + height, + &chain_state, + exit, + )?; + stores.commit( + height, + mem::take(&mut addressindex_to_typedindex_to_sent_outputindex), + mem::take( &mut addressindex_to_typedindex_to_received_outputindex) + )?; exit.release(); } @@ -579,6 +631,13 @@ impl Vecs { info!("Flushing..."); self.flush_states(height, &chain_state, exit)?; + stores.commit( + height, + mem::take(&mut addressindex_to_typedindex_to_sent_outputindex), + mem::take(&mut addressindex_to_typedindex_to_received_outputindex), + )?; + } else { + exit.block(); } info!("Computing overlapping..."); @@ -649,6 +708,8 @@ impl Vecs { Some(&self.height_to_opreturn_supply), )?; + stores.rotate_memtables(); + exit.release(); Ok(()) diff --git a/crates/brk_core/src/groups/addressindex_to_typeindex_to_outputindex.rs b/crates/brk_core/src/groups/addressindex_to_typeindex_to_outputindex.rs new file mode 100644 index 000000000..9b0901907 --- /dev/null +++ b/crates/brk_core/src/groups/addressindex_to_typeindex_to_outputindex.rs @@ -0,0 +1,38 @@ +use std::mem; + +use derive_deref::{Deref, DerefMut}; + +use crate::{OutputIndex, TypeIndex}; + +use super::GroupedByAddressType; + +#[derive(Debug, Default, Deref, DerefMut)] +pub struct AddressIndexToTypeIndedToOutputIndex( + GroupedByAddressType>, +); + +impl AddressIndexToTypeIndedToOutputIndex { + pub fn merge(&mut self, mut other: Self) { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + } + + fn merge_(own: &mut Vec<(TypeIndex, OutputIndex)>, other: &mut Vec<(TypeIndex, OutputIndex)>) { + if own.len() >= other.len() { + own.append(other); + } else { + other.append(own); + mem::swap(own, other); + } + } + + pub fn inner(self) -> GroupedByAddressType> { + self.0 + } +} diff --git a/crates/brk_core/src/groups/by_address_type.rs b/crates/brk_core/src/groups/by_address_type.rs index 57e0eaa1a..10716a1f0 100644 --- a/crates/brk_core/src/groups/by_address_type.rs +++ b/crates/brk_core/src/groups/by_address_type.rs @@ -17,17 +17,17 @@ pub struct GroupedByAddressType { } impl GroupedByAddressType { - pub fn get_mut(&mut self, output_type: OutputType) -> &mut T { - match output_type { - OutputType::P2PK65 => &mut self.p2pk65, - OutputType::P2PK33 => &mut self.p2pk33, - OutputType::P2PKH => &mut self.p2pkh, - OutputType::P2SH => &mut self.p2sh, - OutputType::P2WPKH => &mut self.p2wpkh, - OutputType::P2WSH => &mut self.p2wsh, - OutputType::P2TR => &mut self.p2tr, - OutputType::P2A => &mut self.p2a, - _ => unreachable!(), + pub fn get_mut(&mut self, address_type: OutputType) -> Option<&mut T> { + match address_type { + OutputType::P2PK65 => Some(&mut self.p2pk65), + OutputType::P2PK33 => Some(&mut self.p2pk33), + OutputType::P2PKH => Some(&mut self.p2pkh), + OutputType::P2SH => Some(&mut self.p2sh), + OutputType::P2WPKH => Some(&mut self.p2wpkh), + OutputType::P2WSH => Some(&mut self.p2wsh), + OutputType::P2TR => Some(&mut self.p2tr), + OutputType::P2A => Some(&mut self.p2a), + _ => None, } } diff --git a/crates/brk_core/src/groups/mod.rs b/crates/brk_core/src/groups/mod.rs index 55cd604af..268a79586 100644 --- a/crates/brk_core/src/groups/mod.rs +++ b/crates/brk_core/src/groups/mod.rs @@ -1,4 +1,5 @@ mod address; +mod addressindex_to_typeindex_to_outputindex; mod by_address_type; mod by_date_range; mod by_epoch; @@ -15,6 +16,7 @@ mod filter; mod utxo; pub use address::*; +pub use addressindex_to_typeindex_to_outputindex::*; pub use by_address_type::*; pub use by_date_range::*; pub use by_epoch::*; diff --git a/crates/brk_core/src/structs/addressbytes.rs b/crates/brk_core/src/structs/addressbytes.rs index d22cf485f..cc4125608 100644 --- a/crates/brk_core/src/structs/addressbytes.rs +++ b/crates/brk_core/src/structs/addressbytes.rs @@ -162,7 +162,7 @@ impl fmt::Display for P2PKHBytes { .push_opcode(opcodes::all::OP_CHECKSIG) .into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -192,7 +192,7 @@ impl fmt::Display for P2SHBytes { .push_opcode(opcodes::all::OP_EQUAL) .into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -218,7 +218,7 @@ impl fmt::Display for P2WPKHBytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(0).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -244,7 +244,7 @@ impl fmt::Display for P2WSHBytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(0).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -270,7 +270,7 @@ impl fmt::Display for P2TRBytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(1).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -296,7 +296,7 @@ impl fmt::Display for P2ABytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(1).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } diff --git a/crates/brk_core/src/structs/p2aaddressindex_outputindex.rs b/crates/brk_core/src/structs/p2aaddressindex_outputindex.rs index 50c442dd5..7a63af89b 100644 --- a/crates/brk_core/src/structs/p2aaddressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2aaddressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2AAddressIndex}; @@ -11,6 +11,12 @@ pub struct P2AAddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2AAddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2AAddressIndex::from(value.0), value.1)) + } +} + impl From<(P2AAddressIndex, OutputIndex)> for P2AAddressIndexOutputindex { fn from(value: (P2AAddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_core/src/structs/p2pk33addressindex_outputindex.rs b/crates/brk_core/src/structs/p2pk33addressindex_outputindex.rs index b921c5c8b..a8f73ae79 100644 --- a/crates/brk_core/src/structs/p2pk33addressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2pk33addressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2PK33AddressIndex}; @@ -11,6 +11,12 @@ pub struct P2PK33AddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2PK33AddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2PK33AddressIndex::from(value.0), value.1)) + } +} + impl From<(P2PK33AddressIndex, OutputIndex)> for P2PK33AddressIndexOutputindex { fn from(value: (P2PK33AddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_core/src/structs/p2pk65addressindex_outputindex.rs b/crates/brk_core/src/structs/p2pk65addressindex_outputindex.rs index a19b4d2fc..160e380cc 100644 --- a/crates/brk_core/src/structs/p2pk65addressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2pk65addressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2PK65AddressIndex}; @@ -11,6 +11,12 @@ pub struct P2PK65AddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2PK65AddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2PK65AddressIndex::from(value.0), value.1)) + } +} + impl From<(P2PK65AddressIndex, OutputIndex)> for P2PK65AddressIndexOutputindex { fn from(value: (P2PK65AddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_core/src/structs/p2pkhaddressindex_outputindex.rs b/crates/brk_core/src/structs/p2pkhaddressindex_outputindex.rs index d7e6df5db..ab67bed7f 100644 --- a/crates/brk_core/src/structs/p2pkhaddressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2pkhaddressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2PKHAddressIndex}; @@ -11,6 +11,12 @@ pub struct P2PKHAddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2PKHAddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2PKHAddressIndex::from(value.0), value.1)) + } +} + impl From<(P2PKHAddressIndex, OutputIndex)> for P2PKHAddressIndexOutputindex { fn from(value: (P2PKHAddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_core/src/structs/p2shaddressindex_outputindex.rs b/crates/brk_core/src/structs/p2shaddressindex_outputindex.rs index 2b3650db4..6cfda7793 100644 --- a/crates/brk_core/src/structs/p2shaddressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2shaddressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2SHAddressIndex}; @@ -11,6 +11,12 @@ pub struct P2SHAddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2SHAddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2SHAddressIndex::from(value.0), value.1)) + } +} + impl From<(P2SHAddressIndex, OutputIndex)> for P2SHAddressIndexOutputindex { fn from(value: (P2SHAddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_core/src/structs/p2traddressindex_outputindex.rs b/crates/brk_core/src/structs/p2traddressindex_outputindex.rs index 1cc9647ff..d4fb5c45f 100644 --- a/crates/brk_core/src/structs/p2traddressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2traddressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2TRAddressIndex}; @@ -11,6 +11,12 @@ pub struct P2TRAddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2TRAddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2TRAddressIndex::from(value.0), value.1)) + } +} + impl From<(P2TRAddressIndex, OutputIndex)> for P2TRAddressIndexOutputindex { fn from(value: (P2TRAddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_core/src/structs/p2wpkhaddressindex_outputindex.rs b/crates/brk_core/src/structs/p2wpkhaddressindex_outputindex.rs index c16d91d74..320dc4797 100644 --- a/crates/brk_core/src/structs/p2wpkhaddressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2wpkhaddressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2WPKHAddressIndex}; @@ -11,6 +11,12 @@ pub struct P2WPKHAddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2WPKHAddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2WPKHAddressIndex::from(value.0), value.1)) + } +} + impl From<(P2WPKHAddressIndex, OutputIndex)> for P2WPKHAddressIndexOutputindex { fn from(value: (P2WPKHAddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_core/src/structs/p2wshaddressindex_outputindex.rs b/crates/brk_core/src/structs/p2wshaddressindex_outputindex.rs index d3fb72571..3da1d0152 100644 --- a/crates/brk_core/src/structs/p2wshaddressindex_outputindex.rs +++ b/crates/brk_core/src/structs/p2wshaddressindex_outputindex.rs @@ -1,7 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{copy_first_4bytes, copy_first_8bytes}; +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; use super::{OutputIndex, P2WSHAddressIndex}; @@ -11,6 +11,12 @@ pub struct P2WSHAddressIndexOutputindex { outputindex: OutputIndex, } +impl From<(TypeIndex, OutputIndex)> for P2WSHAddressIndexOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self::from((P2WSHAddressIndex::from(value.0), value.1)) + } +} + impl From<(P2WSHAddressIndex, OutputIndex)> for P2WSHAddressIndexOutputindex { fn from(value: (P2WSHAddressIndex, OutputIndex)) -> Self { Self { diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index 6c8382279..778c3897e 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -252,8 +252,8 @@ impl Stores { } } } else { - self.blockhashprefix_to_height.reset_partition()?; - self.addressbyteshash_to_typeindex.reset_partition()?; + self.blockhashprefix_to_height.reset()?; + self.addressbyteshash_to_typeindex.reset()?; } if starting_indexes.txindex != TxIndex::ZERO { @@ -275,7 +275,7 @@ impl Stores { } }); } else { - self.txidprefix_to_txindex.reset_partition()?; + self.txidprefix_to_txindex.reset()?; } self.commit(starting_indexes.height.decremented().unwrap_or_default())?; diff --git a/crates/brk_interface/src/index.rs b/crates/brk_interface/src/index.rs index a92b8fd90..f09dfe8f3 100644 --- a/crates/brk_interface/src/index.rs +++ b/crates/brk_interface/src/index.rs @@ -188,7 +188,7 @@ impl TryFrom<&str> for Index { impl fmt::Display for Index { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } diff --git a/crates/brk_interface/src/output.rs b/crates/brk_interface/src/output.rs index 4562ed66d..287e6bd23 100644 --- a/crates/brk_interface/src/output.rs +++ b/crates/brk_interface/src/output.rs @@ -36,9 +36,9 @@ impl fmt::Display for Output { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Json(value) => write!(f, "{}", serde_json::to_string_pretty(value).unwrap()), - Self::CSV(string) => write!(f, "{}", string), - Self::TSV(string) => write!(f, "{}", string), - Self::MD(string) => write!(f, "{}", string), + Self::CSV(string) => write!(f, "{string}"), + Self::TSV(string) => write!(f, "{string}"), + Self::MD(string) => write!(f, "{string}"), } } } diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index e95f7b465..ab1862bf7 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -34,7 +34,7 @@ pub fn init(path: Option<&Path>) { .strftime("%Y-%m-%d %H:%M:%S") .to_string(); let level = record.level().as_str().to_lowercase(); - let level = format!("{:5}", level); + let level = format!("{level:5}"); let target = record.target(); let dash = "-"; let args = record.args(); @@ -81,7 +81,7 @@ fn write( dash: impl Display, args: impl Display, ) -> Result<(), std::io::Error> { - writeln!(buf, "{} {} {} {}", date_time, dash, level, args) + writeln!(buf, "{date_time} {dash} {level} {args}") // Don't remove, used to know the target of unwanted logs // writeln!( // buf, diff --git a/crates/brk_server/src/api/interface/bridge.rs b/crates/brk_server/src/api/interface/bridge.rs index eac299108..f97d11dc9 100644 --- a/crates/brk_server/src/api/interface/bridge.rs +++ b/crates/brk_server/src/api/interface/bridge.rs @@ -36,10 +36,9 @@ impl Bridge for Interface<'static> { // File auto-generated, any modifications will be overwritten // -export const VERSION = \"v{}\"; +export const VERSION = \"v{VERSION}\"; -", - VERSION +" ); contents += &indexes diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index b5ffa0fcf..704b6ea7f 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -130,8 +130,8 @@ where pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) { if self.needs(height) { if !self.dels.is_empty() { - // self.dels.remove(&key); - unreachable!("Shouldn't reach this"); + self.dels.remove(&key); + // unreachable!("Shouldn't reach this"); } self.puts.insert(key, value); } @@ -181,19 +181,43 @@ where .map_err(|e| e.into()) } - pub fn reset_partition(&mut self) -> Result<()> { - let partition: TransactionalPartitionHandle = self.partition.take().unwrap(); + pub fn commit_( + &mut self, + height: Height, + remove: impl Iterator, + insert: impl Iterator, + ) -> Result<()> { + if self.has(height) { + return Ok(()); + } - self.keyspace.delete_partition(partition)?; + self.meta.export(self.len(), height)?; - self.keyspace.persist(PersistMode::SyncAll)?; + let mut wtx = self.keyspace.write_tx(); - self.meta.reset(); + let partition = self.partition.as_ref().unwrap(); - let partition = - Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filter_bits)?; + remove.for_each(|key| wtx.remove(partition, ByteView::from(key))); - self.partition.replace(partition); + insert.for_each(|(key, value)| { + // if CHECK_COLLISIONS { + // #[allow(unused_must_use)] + // if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) { + // dbg!( + // &key, + // V::try_from(value.as_bytes().into()).unwrap(), + // &self.meta, + // self.rtx.get(&self.partition, key.as_bytes()) + // ); + // unreachable!(); + // } + // } + wtx.insert(partition, ByteView::from(key), ByteView::from(value)) + }); + + wtx.commit()?; + + self.rtx = self.keyspace.read_tx(); Ok(()) } @@ -206,41 +230,29 @@ where ByteView: From + From<&'a K> + From, { fn commit(&mut self, height: Height) -> Result<()> { - if self.has(height) && self.puts.is_empty() && self.dels.is_empty() { + if self.puts.is_empty() && self.dels.is_empty() { return Ok(()); } - self.meta.export(self.len(), height)?; + let dels = mem::take(&mut self.dels); + let puts = mem::take(&mut self.puts); - let mut wtx = self.keyspace.write_tx(); + self.commit_(height, dels.into_iter(), puts.into_iter()) + } - let partition = self.partition.as_ref().unwrap(); + fn reset(&mut self) -> Result<()> { + let partition: TransactionalPartitionHandle = self.partition.take().unwrap(); - mem::take(&mut self.dels) - .into_iter() - .for_each(|key| wtx.remove(partition, ByteView::from(key))); + self.keyspace.delete_partition(partition)?; - mem::take(&mut self.puts) - .into_iter() - .for_each(|(key, value)| { - // if CHECK_COLLISIONS { - // #[allow(unused_must_use)] - // if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) { - // dbg!( - // &key, - // V::try_from(value.as_bytes().into()).unwrap(), - // &self.meta, - // self.rtx.get(&self.partition, key.as_bytes()) - // ); - // unreachable!(); - // } - // } - wtx.insert(partition, ByteView::from(key), ByteView::from(value)) - }); + self.keyspace.persist(PersistMode::SyncAll)?; - wtx.commit()?; + self.meta.reset(); - self.rtx = self.keyspace.read_tx(); + let partition = + Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filter_bits)?; + + self.partition.replace(partition); Ok(()) } @@ -274,6 +286,10 @@ where fn needs(&self, height: Height) -> bool { self.meta.needs(height) } + + fn version(&self) -> Version { + self.meta.version() + } } impl Clone for Store diff --git a/crates/brk_store/src/meta.rs b/crates/brk_store/src/meta.rs index bf052f9a4..4360249c9 100644 --- a/crates/brk_store/src/meta.rs +++ b/crates/brk_store/src/meta.rs @@ -66,9 +66,9 @@ impl StoreMeta { // self.len() == 0 // } - // pub fn version(&self) -> Version { - // self.version - // } + pub fn version(&self) -> Version { + self.version + } pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> { self.len = len; diff --git a/crates/brk_store/src/trait.rs b/crates/brk_store/src/trait.rs index 2bd0b1268..90379fd96 100644 --- a/crates/brk_store/src/trait.rs +++ b/crates/brk_store/src/trait.rs @@ -1,8 +1,10 @@ -use brk_core::{Height, Result}; +use brk_core::{Height, Result, Version}; pub trait AnyStore { fn commit(&mut self, height: Height) -> Result<()>; + fn reset(&mut self) -> Result<()>; + fn rotate_memtable(&self); fn height(&self) -> Option; @@ -16,4 +18,6 @@ pub trait AnyStore { fn has(&self, height: Height) -> bool; fn needs(&self, height: Height) -> bool; + + fn version(&self) -> Version; }