diff --git a/Cargo.lock b/Cargo.lock index 4e3ec80dc..666f88e90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,22 +122,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -228,9 +228,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "axum" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" +checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" dependencies = [ "axum-core", "bytes", @@ -667,7 +667,9 @@ dependencies = [ [[package]] name = "brk_fjall" -version = "2.11.5" +version = "2.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da285ef974591f84284f6ab500dfd903c7a9bf8674e2c17f8b35a44a20726ee1" dependencies = [ "byteorder", "byteview 0.6.1", @@ -1294,7 +1296,7 @@ version = "0.0.111" dependencies = [ "bitcoin", "brk_error", - "byteview 0.8.0", + "byteview 0.6.1", "derive_deref", "itoa", "jiff", @@ -1360,9 +1362,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "byteview" @@ -1387,9 +1389,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.45" +version = "1.2.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36" dependencies = [ "find-msvc-tools", "jobserver", @@ -2104,9 +2106,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fixedbitset" @@ -2528,9 +2530,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", @@ -2549,9 +2551,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ "bytes", "futures-core", @@ -4375,18 +4377,18 @@ dependencies = [ [[package]] name = "rolldown-file-id" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e8a353a557a02405575e020bf99e1be57e74fc8293c400d04c3cd550e5d5b41" +checksum = "987e6977d7746be897071f69d1f8ec72b25a246c8130cf7985c1a536f6eaf1be" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "rolldown-notify" -version = "8.2.0" +version = "8.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97af9361dbd2e8380d54cebb6b005b09998883ee2eedbded4edf159f59357936" +checksum = "858a3e78503d89287b47d66fb042b4f04e34b237774506fbb48d7c3e5eb445cf" dependencies = [ "bitflags 2.10.0", "fsevent-sys", @@ -4397,14 +4399,14 @@ dependencies = [ "mio", "rolldown-notify-types", "walkdir", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "rolldown-notify-debouncer-full" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa96424871058b73046901eef2ef31bf3bf5e46880476e7a1debfc8f34aea6" +checksum = "45e06775bbad4fa50fb0c07a1dfbb5ec3356131b6f0e19679910cb698aa54068" dependencies = [ "log", "rolldown-file-id", @@ -4415,9 +4417,9 @@ dependencies = [ [[package]] name = "rolldown-notify-types" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35fd0bfc4c6e8d4f0f7f69508309e97d8db39638ec0093577c77748be2b7ae8" +checksum = "598d3e49d0f0fb6f5be6664efc3138634d7aa0c9fc5d1186a0a435dd76e6660f" [[package]] name = "ropey" diff --git a/Cargo.toml b/Cargo.toml index eb741e44e..bff53cb96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ inherits = "release" [workspace.dependencies] aide = { version = "0.16.0-alpha.1", features = ["axum-json", "axum-query"] } -axum = "0.8.6" +axum = "0.8.7" bitcoin = { version = "0.32.7", features = ["serde"] } bitcoincore-rpc = "0.19.0" brk_bencher = { version = "0.0.111", path = "crates/brk_bencher" } @@ -58,11 +58,11 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" } brk_types = { version = "0.0.111", path = "crates/brk_types" } brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["derive"] } brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" } -# byteview = "=0.6.1" -byteview = "~0.8.0" +byteview = "=0.6.1" +# byteview = "~0.8.0" derive_deref = "1.1.1" -# fjall2 = { version = "2.11.5", package = "brk_fjall" } -fjall2 = { path = "../fjall2", package = "brk_fjall" } +fjall2 = { version = "2.11.8", package = "brk_fjall" } +# fjall2 = { path = "../fjall2", package = "brk_fjall" } # fjall2 = { version = "2.11.2", package = "fjall" } # fjall3 = { version = "=3.0.0-pre.5", package = "fjall" } fjall3 = { path = "../fjall3", package = "fjall" } diff --git a/crates/brk_computer/examples/computer_bench.rs b/crates/brk_computer/examples/computer_bench.rs index 729685ab5..db0508899 100644 --- a/crates/brk_computer/examples/computer_bench.rs +++ b/crates/brk_computer/examples/computer_bench.rs @@ -4,7 +4,7 @@ use brk_bencher::Bencher; use brk_computer::Computer; use brk_error::Result; use brk_fetcher::Fetcher; -use brk_indexer::Indexer; +use brk_indexer::{Indexer, Indexes}; use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; @@ -58,7 +58,8 @@ fn run() -> Result<()> { }); let i = Instant::now(); - let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?; + // let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?; + let starting_indexes = Indexes::default(); info!("Done in {:?}", i.elapsed()); let i = Instant::now(); diff --git a/crates/brk_computer/src/blks.rs b/crates/brk_computer/src/blks.rs index c170066e1..f0bcb6ebe 100644 --- a/crates/brk_computer/src/blks.rs +++ b/crates/brk_computer/src/blks.rs @@ -97,7 +97,7 @@ impl Vecs { let height = block.height(); self.height_to_position - .forced_push(height, block.metadata().position(), exit)?; + .truncate_push(height, block.metadata().position())?; let txindex = height_to_first_txindex_iter.get_unwrap(height); @@ -105,7 +105,7 @@ impl Vecs { |(index, metadata)| -> Result<()> { let txindex = txindex + index; self.txindex_to_position - .forced_push(txindex, metadata.position(), exit)?; + .truncate_push(txindex, metadata.position())?; Ok(()) }, )?; diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index 3ce35ae31..d84fd9cc7 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -85,12 +85,11 @@ impl Vecs { .enumerate() .skip(index.to_usize()) .try_for_each(|(i, v)| -> Result<()> { - self.height_to_price_ohlc_in_cents.forced_push_at( + self.height_to_price_ohlc_in_cents.truncate_push_at( i, self.fetcher .get_height(i.into(), v, prev_timestamp) .unwrap(), - exit, )?; prev_timestamp = Some(v); Ok(()) @@ -127,7 +126,7 @@ impl Vecs { prev.replace(ohlc.clone()); self.dateindex_to_price_ohlc_in_cents - .forced_push_at(i, ohlc, exit)?; + .truncate_push_at(i, ohlc)?; Ok(()) })?; diff --git a/crates/brk_computer/src/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs index 9d3d13184..b728ef1dd 100644 --- a/crates/brk_computer/src/grouped/builder_eager.rs +++ b/crates/brk_computer/src/grouped/builder_eager.rs @@ -226,7 +226,7 @@ where .skip(index.to_usize()) .try_for_each(|(i, v)| -> Result<()> { cumulative += v; - cumulative_vec.forced_push_at(i, cumulative, exit)?; + cumulative_vec.truncate_push_at(i, cumulative)?; Ok(()) })?; @@ -246,6 +246,10 @@ where where I2: VecIndex + VecValue + CheckedSub, { + dbg!(source.len()); + dbg!(first_indexes.len()); + dbg!(count_indexes.len()); + self.validate_computed_version_or_reset( source.version() + first_indexes.version() + count_indexes.version(), )?; @@ -275,7 +279,7 @@ where let f = source_iter .get(first_index) .unwrap_or_else(|| T::from(0_usize)); - first.forced_push_at(index, f, exit)?; + first.truncate_push_at(index, f)?; } if let Some(last) = self.last.as_mut() { @@ -291,7 +295,7 @@ where // }) // .unwrap() // ; - last.forced_push_at(index, v, exit)?; + last.truncate_push_at(index, v)?; } let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some(); @@ -316,7 +320,7 @@ where values.sort_unstable(); if let Some(max) = self.max.as_mut() { - max.forced_push_at( + max.truncate_push_at( index, *values .last() @@ -325,6 +329,7 @@ where dbg!( &values, max.name(), + index, first_indexes.name(), first_index, count_indexes.name(), @@ -334,32 +339,31 @@ where ); }) .unwrap(), - exit, )?; } if let Some(pct90) = self.pct90.as_mut() { - pct90.forced_push_at(index, get_percentile(&values, 0.90), exit)?; + pct90.truncate_push_at(index, get_percentile(&values, 0.90))?; } if let Some(pct75) = self.pct75.as_mut() { - pct75.forced_push_at(index, get_percentile(&values, 0.75), exit)?; + pct75.truncate_push_at(index, get_percentile(&values, 0.75))?; } if let Some(median) = self.median.as_mut() { - median.forced_push_at(index, get_percentile(&values, 0.50), exit)?; + median.truncate_push_at(index, get_percentile(&values, 0.50))?; } if let Some(pct25) = self.pct25.as_mut() { - pct25.forced_push_at(index, get_percentile(&values, 0.25), exit)?; + pct25.truncate_push_at(index, get_percentile(&values, 0.25))?; } if let Some(pct10) = self.pct10.as_mut() { - pct10.forced_push_at(index, get_percentile(&values, 0.10), exit)?; + pct10.truncate_push_at(index, get_percentile(&values, 0.10))?; } if let Some(min) = self.min.as_mut() { - min.forced_push_at(index, *values.first().unwrap(), exit)?; + min.truncate_push_at(index, *values.first().unwrap())?; } } @@ -369,18 +373,18 @@ where if let Some(average) = self.average.as_mut() { let avg = sum / len; - average.forced_push_at(index, avg, exit)?; + average.truncate_push_at(index, avg)?; } if needs_sum_or_cumulative { if let Some(sum_vec) = self.sum.as_mut() { - sum_vec.forced_push_at(index, sum, exit)?; + sum_vec.truncate_push_at(index, sum)?; } if let Some(cumulative_vec) = self.cumulative.as_mut() { let t = cumulative.unwrap() + sum; cumulative.replace(t); - cumulative_vec.forced_push_at(index, t, exit)?; + cumulative_vec.truncate_push_at(index, t)?; } } } @@ -445,7 +449,7 @@ where if let Some(first) = self.first.as_mut() { let v = source_first_iter.as_mut().unwrap().get_unwrap(first_index); - first.forced_push_at(index, v, exit)?; + first.truncate_push_at(index, v)?; } if let Some(last) = self.last.as_mut() { @@ -455,7 +459,7 @@ where } let last_index = first_index + (count_index - 1); let v = source_last_iter.as_mut().unwrap().get_unwrap(last_index); - last.forced_push_at(index, v, exit)?; + last.truncate_push_at(index, v)?; } let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some(); @@ -473,7 +477,7 @@ where .take(*count_index as usize) .collect::>(); values.sort_unstable(); - max.forced_push_at(index, *values.last().unwrap(), exit)?; + max.truncate_push_at(index, *values.last().unwrap())?; } if let Some(min) = self.min.as_mut() { @@ -483,7 +487,7 @@ where .take(*count_index as usize) .collect::>(); values.sort_unstable(); - min.forced_push_at(index, *values.first().unwrap(), exit)?; + min.truncate_push_at(index, *values.first().unwrap())?; } } @@ -500,7 +504,7 @@ where // TODO: Multiply by count then divide by cumulative // Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example) let avg = cumulative / len; - average.forced_push_at(index, avg, exit)?; + average.truncate_push_at(index, avg)?; } if needs_sum_or_cumulative { @@ -513,13 +517,13 @@ where let sum = values.into_iter().fold(T::from(0), |a, b| a + b); if let Some(sum_vec) = self.sum.as_mut() { - sum_vec.forced_push_at(index, sum, exit)?; + sum_vec.truncate_push_at(index, sum)?; } if let Some(cumulative_vec) = self.cumulative.as_mut() { let t = cumulative.unwrap() + sum; cumulative.replace(t); - cumulative_vec.forced_push_at(index, t, exit)?; + cumulative_vec.truncate_push_at(index, t)?; } } } diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index 3a74b94d9..58bc445dc 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -252,84 +252,73 @@ impl ComputedVecsFromTxindex { .map(Height::from) .try_for_each(|height| -> Result<()> { if let Some(first) = self.height.first.as_mut() { - first.forced_push( + first.truncate_push( height, Bitcoin::from(sats.height.unwrap_first().into_iter().get_unwrap(height)), - exit, )?; } if let Some(average) = self.height.average.as_mut() { - average.forced_push( + average.truncate_push( height, Bitcoin::from(sats.height.unwrap_average().into_iter().get_unwrap(height)), - exit, )?; } if let Some(sum) = self.height.sum.as_mut() { - sum.forced_push( + sum.truncate_push( height, Bitcoin::from(sats.height.unwrap_sum().into_iter().get_unwrap(height)), - exit, )?; } if let Some(max) = self.height.max.as_mut() { - max.forced_push( + max.truncate_push( height, Bitcoin::from(sats.height.unwrap_max().into_iter().get_unwrap(height)), - exit, )?; } if let Some(pct90) = self.height.pct90.as_mut() { - pct90.forced_push( + pct90.truncate_push( height, Bitcoin::from(sats.height.unwrap_pct90().into_iter().get_unwrap(height)), - exit, )?; } if let Some(pct75) = self.height.pct75.as_mut() { - pct75.forced_push( + pct75.truncate_push( height, Bitcoin::from(sats.height.unwrap_pct75().into_iter().get_unwrap(height)), - exit, )?; } if let Some(median) = self.height.median.as_mut() { - median.forced_push( + median.truncate_push( height, Bitcoin::from(sats.height.unwrap_median().into_iter().get_unwrap(height)), - exit, )?; } if let Some(pct25) = self.height.pct25.as_mut() { - pct25.forced_push( + pct25.truncate_push( height, Bitcoin::from(sats.height.unwrap_pct25().into_iter().get_unwrap(height)), - exit, )?; } if let Some(pct10) = self.height.pct10.as_mut() { - pct10.forced_push( + pct10.truncate_push( height, Bitcoin::from(sats.height.unwrap_pct10().into_iter().get_unwrap(height)), - exit, )?; } if let Some(min) = self.height.min.as_mut() { - min.forced_push( + min.truncate_push( height, Bitcoin::from(sats.height.unwrap_min().into_iter().get_unwrap(height)), - exit, )?; } if let Some(last) = self.height.last.as_mut() { - last.forced_push( + last.truncate_push( height, Bitcoin::from(sats.height.unwrap_last().into_iter().get_unwrap(height)), - exit, )?; } if let Some(cumulative) = self.height.cumulative.as_mut() { - cumulative.forced_push( + cumulative.truncate_push( height, Bitcoin::from( sats.height @@ -337,7 +326,6 @@ impl ComputedVecsFromTxindex { .into_iter() .get_unwrap(height), ), - exit, )?; } Ok(()) @@ -380,14 +368,13 @@ impl ComputedVecsFromTxindex { let price = *close_iter.get_unwrap(height); if let Some(first) = self.height.first.as_mut() { - first.forced_push( + first.truncate_push( height, price * bitcoin.height.unwrap_first().into_iter().get_unwrap(height), - exit, )?; } if let Some(average) = self.height.average.as_mut() { - average.forced_push( + average.truncate_push( height, price * bitcoin @@ -395,39 +382,34 @@ impl ComputedVecsFromTxindex { .unwrap_average() .into_iter() .get_unwrap(height), - exit, )?; } if let Some(sum) = self.height.sum.as_mut() { - sum.forced_push( + sum.truncate_push( height, price * bitcoin.height.unwrap_sum().into_iter().get_unwrap(height), - exit, )?; } if let Some(max) = self.height.max.as_mut() { - max.forced_push( + max.truncate_push( height, price * bitcoin.height.unwrap_max().into_iter().get_unwrap(height), - exit, )?; } if let Some(pct90) = self.height.pct90.as_mut() { - pct90.forced_push( + pct90.truncate_push( height, price * bitcoin.height.unwrap_pct90().into_iter().get_unwrap(height), - exit, )?; } if let Some(pct75) = self.height.pct75.as_mut() { - pct75.forced_push( + pct75.truncate_push( height, price * bitcoin.height.unwrap_pct75().into_iter().get_unwrap(height), - exit, )?; } if let Some(median) = self.height.median.as_mut() { - median.forced_push( + median.truncate_push( height, price * bitcoin @@ -435,39 +417,34 @@ impl ComputedVecsFromTxindex { .unwrap_median() .into_iter() .get_unwrap(height), - exit, )?; } if let Some(pct25) = self.height.pct25.as_mut() { - pct25.forced_push( + pct25.truncate_push( height, price * bitcoin.height.unwrap_pct25().into_iter().get_unwrap(height), - exit, )?; } if let Some(pct10) = self.height.pct10.as_mut() { - pct10.forced_push( + pct10.truncate_push( height, price * bitcoin.height.unwrap_pct10().into_iter().get_unwrap(height), - exit, )?; } if let Some(min) = self.height.min.as_mut() { - min.forced_push( + min.truncate_push( height, price * bitcoin.height.unwrap_min().into_iter().get_unwrap(height), - exit, )?; } if let Some(last) = self.height.last.as_mut() { - last.forced_push( + last.truncate_push( height, price * bitcoin.height.unwrap_last().into_iter().get_unwrap(height), - exit, )?; } if let Some(cumulative) = self.height.cumulative.as_mut() { - cumulative.forced_push( + cumulative.truncate_push( height, price * bitcoin @@ -475,7 +452,6 @@ impl ComputedVecsFromTxindex { .unwrap_cumulative() .into_iter() .get_unwrap(height), - exit, )?; } Ok(()) diff --git a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs index 06ef8dc01..11a8bac35 100644 --- a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs @@ -407,42 +407,42 @@ impl ComputedRatioVecsFromDateIndex { .dateindex .as_mut() .unwrap() - .forced_push_at(index, StoredF32::NAN, exit)?; + .truncate_push_at(index, StoredF32::NAN)?; self.ratio_pct2 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, StoredF32::NAN, exit)?; + .truncate_push_at(index, StoredF32::NAN)?; self.ratio_pct1 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, StoredF32::NAN, exit)?; + .truncate_push_at(index, StoredF32::NAN)?; self.ratio_pct95 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, StoredF32::NAN, exit)?; + .truncate_push_at(index, StoredF32::NAN)?; self.ratio_pct98 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, StoredF32::NAN, exit)?; + .truncate_push_at(index, StoredF32::NAN)?; self.ratio_pct99 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, StoredF32::NAN, exit)?; + .truncate_push_at(index, StoredF32::NAN)?; } else { let pos = sorted.binary_search(&ratio).unwrap_or_else(|pos| pos); sorted.insert(pos, ratio); @@ -453,42 +453,42 @@ impl ComputedRatioVecsFromDateIndex { .dateindex .as_mut() .unwrap() - .forced_push_at(index, get_percentile(&sorted, 0.01), exit)?; + .truncate_push_at(index, get_percentile(&sorted, 0.01))?; self.ratio_pct2 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, get_percentile(&sorted, 0.02), exit)?; + .truncate_push_at(index, get_percentile(&sorted, 0.02))?; self.ratio_pct5 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, get_percentile(&sorted, 0.05), exit)?; + .truncate_push_at(index, get_percentile(&sorted, 0.05))?; self.ratio_pct95 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, get_percentile(&sorted, 0.95), exit)?; + .truncate_push_at(index, get_percentile(&sorted, 0.95))?; self.ratio_pct98 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, get_percentile(&sorted, 0.98), exit)?; + .truncate_push_at(index, get_percentile(&sorted, 0.98))?; self.ratio_pct99 .as_mut() .unwrap() .dateindex .as_mut() .unwrap() - .forced_push_at(index, get_percentile(&sorted, 0.99), exit)?; + .truncate_push_at(index, get_percentile(&sorted, 0.99))?; } Ok(()) diff --git a/crates/brk_computer/src/grouped/sd_from_dateindex.rs b/crates/brk_computer/src/grouped/sd_from_dateindex.rs index 87e46d9e8..182196b33 100644 --- a/crates/brk_computer/src/grouped/sd_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/sd_from_dateindex.rs @@ -505,47 +505,47 @@ impl ComputedStandardDeviationVecsFromDateIndex { .skip(starting_dateindex.to_usize()) .try_for_each(|(index, ratio)| -> Result<()> { if index < min_date_usize { - self.sd.dateindex.as_mut().unwrap().forced_push_at( - index, - StoredF32::NAN, - exit, - )?; + self.sd + .dateindex + .as_mut() + .unwrap() + .truncate_push_at(index, StoredF32::NAN)?; if let Some(v) = p0_5sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = p1sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = p1_5sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = p2sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = p2_5sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = p3sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = m0_5sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = m1sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = m1_5sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = m2sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = m2_5sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } if let Some(v) = m3sd.as_mut() { - v.forced_push_at(index, StoredF32::NAN, exit)? + v.truncate_push_at(index, StoredF32::NAN)? } // Advance iterator to stay in sync sma_iter.next(); @@ -567,42 +567,42 @@ impl ComputedStandardDeviationVecsFromDateIndex { .dateindex .as_mut() .unwrap() - .forced_push_at(index, sd, exit)?; + .truncate_push_at(index, sd)?; if let Some(v) = p0_5sd.as_mut() { - v.forced_push_at(index, avg + StoredF32::from(0.5 * *sd), exit)? + v.truncate_push_at(index, avg + StoredF32::from(0.5 * *sd))? } if let Some(v) = p1sd.as_mut() { - v.forced_push_at(index, avg + sd, exit)? + v.truncate_push_at(index, avg + sd)? } if let Some(v) = p1_5sd.as_mut() { - v.forced_push_at(index, avg + StoredF32::from(1.5 * *sd), exit)? + v.truncate_push_at(index, avg + StoredF32::from(1.5 * *sd))? } if let Some(v) = p2sd.as_mut() { - v.forced_push_at(index, avg + 2 * sd, exit)? + v.truncate_push_at(index, avg + 2 * sd)? } if let Some(v) = p2_5sd.as_mut() { - v.forced_push_at(index, avg + StoredF32::from(2.5 * *sd), exit)? + v.truncate_push_at(index, avg + StoredF32::from(2.5 * *sd))? } if let Some(v) = p3sd.as_mut() { - v.forced_push_at(index, avg + 3 * sd, exit)? + v.truncate_push_at(index, avg + 3 * sd)? } if let Some(v) = m0_5sd.as_mut() { - v.forced_push_at(index, avg - StoredF32::from(0.5 * *sd), exit)? + v.truncate_push_at(index, avg - StoredF32::from(0.5 * *sd))? } if let Some(v) = m1sd.as_mut() { - v.forced_push_at(index, avg - sd, exit)? + v.truncate_push_at(index, avg - sd)? } if let Some(v) = m1_5sd.as_mut() { - v.forced_push_at(index, avg - StoredF32::from(1.5 * *sd), exit)? + v.truncate_push_at(index, avg - StoredF32::from(1.5 * *sd))? } if let Some(v) = m2sd.as_mut() { - v.forced_push_at(index, avg - 2 * sd, exit)? + v.truncate_push_at(index, avg - 2 * sd)? } if let Some(v) = m2_5sd.as_mut() { - v.forced_push_at(index, avg - StoredF32::from(2.5 * *sd), exit)? + v.truncate_push_at(index, avg - StoredF32::from(2.5 * *sd))? } if let Some(v) = m3sd.as_mut() { - v.forced_push_at(index, avg - 3 * sd, exit)? + v.truncate_push_at(index, avg - 3 * sd)? } } diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 357edd3e1..281c27630 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -157,6 +157,8 @@ impl Computer { info!("Computing indexes..."); let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; + return Ok(()); + if let Some(fetched) = self.fetched.as_mut() { info!("Computing fetched..."); fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?; @@ -213,8 +215,6 @@ impl Computer { exit, )?; - return Ok(()); - info!("Computing stateful..."); self.stateful.compute( indexer, diff --git a/crates/brk_computer/src/stateful/address_cohort.rs b/crates/brk_computer/src/stateful/address_cohort.rs index 1058d847c..c0e747680 100644 --- a/crates/brk_computer/src/stateful/address_cohort.rs +++ b/crates/brk_computer/src/stateful/address_cohort.rs @@ -127,35 +127,30 @@ impl DynCohortVecs for Vecs { self.inner.validate_computed_versions(base_version) } - fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { + fn truncate_push(&mut self, height: Height) -> Result<()> { if self.starting_height.unwrap() > height { return Ok(()); } - self.height_to_addr_count.forced_push( - height, - self.state.as_ref().unwrap().addr_count.into(), - exit, - )?; + self.height_to_addr_count + .truncate_push(height, self.state.as_ref().unwrap().addr_count.into())?; self.inner - .forced_pushed_at(height, exit, &self.state.as_ref().unwrap().inner) + .truncate_push(height, &self.state.as_ref().unwrap().inner) } - fn compute_then_force_push_unrealized_states( + fn compute_then_truncate_push_unrealized_states( &mut self, height: Height, height_price: Option, dateindex: Option, date_price: Option>, - exit: &Exit, ) -> Result<()> { - self.inner.compute_then_force_push_unrealized_states( + self.inner.compute_then_truncate_push_unrealized_states( height, height_price, dateindex, date_price, - exit, &self.state.as_ref().unwrap().inner, ) } diff --git a/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs index bf609b841..4e5276154 100644 --- a/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs @@ -3,7 +3,7 @@ use brk_grouper::ByAddressType; use brk_traversable::Traversable; use brk_types::{Height, StoredU64}; use derive_deref::{Deref, DerefMut}; -use vecdb::{EagerVec, Exit, GenericStoredVec}; +use vecdb::{EagerVec, GenericStoredVec}; use super::AddressTypeToAddressCount; @@ -18,28 +18,27 @@ impl From>> for AddressTypeToHeightToA } impl AddressTypeToHeightToAddressCount { - pub fn forced_push( + pub fn truncate_push( &mut self, height: Height, addresstype_to_usize: &AddressTypeToAddressCount, - exit: &Exit, ) -> Result<()> { self.p2pk65 - .forced_push(height, addresstype_to_usize.p2pk65.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2pk65.into())?; self.p2pk33 - .forced_push(height, addresstype_to_usize.p2pk33.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2pk33.into())?; self.p2pkh - .forced_push(height, addresstype_to_usize.p2pkh.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2pkh.into())?; self.p2sh - .forced_push(height, addresstype_to_usize.p2sh.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2sh.into())?; self.p2wpkh - .forced_push(height, addresstype_to_usize.p2wpkh.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2wpkh.into())?; self.p2wsh - .forced_push(height, addresstype_to_usize.p2wsh.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2wsh.into())?; self.p2tr - .forced_push(height, addresstype_to_usize.p2tr.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2tr.into())?; self.p2a - .forced_push(height, addresstype_to_usize.p2a.into(), exit)?; + .truncate_push(height, addresstype_to_usize.p2a.into())?; Ok(()) } diff --git a/crates/brk_computer/src/stateful/common.rs b/crates/brk_computer/src/stateful/common.rs index 57c9693ee..a61d0a0d1 100644 --- a/crates/brk_computer/src/stateful/common.rs +++ b/crates/brk_computer/src/stateful/common.rs @@ -1564,26 +1564,18 @@ impl Vecs { Ok(()) } - pub fn forced_pushed_at( - &mut self, - height: Height, - exit: &Exit, - state: &CohortState, - ) -> Result<()> { + pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> { self.height_to_supply - .forced_push(height, state.supply.value, exit)?; + .truncate_push(height, state.supply.value)?; - self.height_to_utxo_count.forced_push( - height, - StoredU64::from(state.supply.utxo_count), - exit, - )?; + self.height_to_utxo_count + .truncate_push(height, StoredU64::from(state.supply.utxo_count))?; self.height_to_satblocks_destroyed - .forced_push(height, state.satblocks_destroyed, exit)?; + .truncate_push(height, state.satblocks_destroyed)?; self.height_to_satdays_destroyed - .forced_push(height, state.satdays_destroyed, exit)?; + .truncate_push(height, state.satdays_destroyed)?; if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { let realized = state.realized.as_ref().unwrap_or_else(|| { @@ -1591,72 +1583,67 @@ impl Vecs { panic!(); }); - height_to_realized_cap.forced_push(height, realized.cap, exit)?; + height_to_realized_cap.truncate_push(height, realized.cap)?; self.height_to_realized_profit .as_mut() .unwrap() - .forced_push(height, realized.profit, exit)?; - self.height_to_realized_loss.as_mut().unwrap().forced_push( - height, - realized.loss, - exit, - )?; - self.height_to_value_created.as_mut().unwrap().forced_push( - height, - realized.value_created, - exit, - )?; + .truncate_push(height, realized.profit)?; + self.height_to_realized_loss + .as_mut() + .unwrap() + .truncate_push(height, realized.loss)?; + self.height_to_value_created + .as_mut() + .unwrap() + .truncate_push(height, realized.value_created)?; self.height_to_value_destroyed .as_mut() .unwrap() - .forced_push(height, realized.value_destroyed, exit)?; + .truncate_push(height, realized.value_destroyed)?; if self.height_to_adjusted_value_created.is_some() { self.height_to_adjusted_value_created .as_mut() .unwrap() - .forced_push(height, realized.adj_value_created, exit)?; + .truncate_push(height, realized.adj_value_created)?; self.height_to_adjusted_value_destroyed .as_mut() .unwrap() - .forced_push(height, realized.adj_value_destroyed, exit)?; + .truncate_push(height, realized.adj_value_destroyed)?; } } Ok(()) } - pub fn compute_then_force_push_unrealized_states( + pub fn compute_then_truncate_push_unrealized_states( &mut self, height: Height, height_price: Option, dateindex: Option, date_price: Option>, - exit: &Exit, state: &CohortState, ) -> Result<()> { if let Some(height_price) = height_price { self.height_to_min_price_paid .as_mut() .unwrap() - .forced_push( + .truncate_push( height, state .price_to_amount_first_key_value() .map(|(&dollars, _)| dollars) .unwrap_or(Dollars::NAN), - exit, )?; self.height_to_max_price_paid .as_mut() .unwrap() - .forced_push( + .truncate_push( height, state .price_to_amount_last_key_value() .map(|(&dollars, _)| dollars) .unwrap_or(Dollars::NAN), - exit, )?; let (height_unrealized_state, date_unrealized_state) = @@ -1665,19 +1652,19 @@ impl Vecs { self.height_to_supply_in_profit .as_mut() .unwrap() - .forced_push(height, height_unrealized_state.supply_in_profit, exit)?; + .truncate_push(height, height_unrealized_state.supply_in_profit)?; self.height_to_supply_in_loss .as_mut() .unwrap() - .forced_push(height, height_unrealized_state.supply_in_loss, exit)?; + .truncate_push(height, height_unrealized_state.supply_in_loss)?; self.height_to_unrealized_profit .as_mut() .unwrap() - .forced_push(height, height_unrealized_state.unrealized_profit, exit)?; + .truncate_push(height, height_unrealized_state.unrealized_profit)?; self.height_to_unrealized_loss .as_mut() .unwrap() - .forced_push(height, height_unrealized_state.unrealized_loss, exit)?; + .truncate_push(height, height_unrealized_state.unrealized_loss)?; if let Some(date_unrealized_state) = date_unrealized_state { let dateindex = dateindex.unwrap(); @@ -1685,19 +1672,19 @@ impl Vecs { self.dateindex_to_supply_in_profit .as_mut() .unwrap() - .forced_push(dateindex, date_unrealized_state.supply_in_profit, exit)?; + .truncate_push(dateindex, date_unrealized_state.supply_in_profit)?; self.dateindex_to_supply_in_loss .as_mut() .unwrap() - .forced_push(dateindex, date_unrealized_state.supply_in_loss, exit)?; + .truncate_push(dateindex, date_unrealized_state.supply_in_loss)?; self.dateindex_to_unrealized_profit .as_mut() .unwrap() - .forced_push(dateindex, date_unrealized_state.unrealized_profit, exit)?; + .truncate_push(dateindex, date_unrealized_state.unrealized_profit)?; self.dateindex_to_unrealized_loss .as_mut() .unwrap() - .forced_push(dateindex, date_unrealized_state.unrealized_loss, exit)?; + .truncate_push(dateindex, date_unrealized_state.unrealized_loss)?; } } diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 5833de367..9482c0ea1 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -1241,22 +1241,16 @@ impl Vecs { }); self.height_to_unspendable_supply - .forced_push(height, unspendable_supply, exit)?; + .truncate_push(height, unspendable_supply)?; self.height_to_opreturn_supply - .forced_push(height, opreturn_supply, exit)?; + .truncate_push(height, opreturn_supply)?; - self.addresstype_to_height_to_addr_count.forced_push( - height, - &addresstype_to_addr_count, - exit, - )?; + self.addresstype_to_height_to_addr_count + .truncate_push(height, &addresstype_to_addr_count)?; - self.addresstype_to_height_to_empty_addr_count.forced_push( - height, - &addresstype_to_empty_addr_count, - exit, - )?; + self.addresstype_to_height_to_empty_addr_count + .truncate_push(height, &addresstype_to_empty_addr_count)?; let date = height_to_date_fixed_iter.get_unwrap(height); let dateindex = DateIndex::try_from(date).unwrap(); @@ -1280,9 +1274,9 @@ impl Vecs { .map(|Filtered(_, v)| v as &mut dyn DynCohortVecs), ) .try_for_each(|v| { - v.forced_pushed_at(height, exit)?; - v.compute_then_force_push_unrealized_states( - height, price, dateindex, date_price, exit, + v.truncate_push(height)?; + v.compute_then_truncate_push_unrealized_states( + height, price, dateindex, date_price, ) })?; diff --git a/crates/brk_computer/src/stateful/trait.rs b/crates/brk_computer/src/stateful/trait.rs index 354e6ad98..64005bbae 100644 --- a/crates/brk_computer/src/stateful/trait.rs +++ b/crates/brk_computer/src/stateful/trait.rs @@ -12,15 +12,14 @@ pub trait DynCohortVecs: Send + Sync { fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>; - fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()>; + fn truncate_push(&mut self, height: Height) -> Result<()>; - fn compute_then_force_push_unrealized_states( + fn compute_then_truncate_push_unrealized_states( &mut self, height: Height, height_price: Option, dateindex: Option, date_price: Option>, - exit: &Exit, ) -> Result<()>; fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>; diff --git a/crates/brk_computer/src/stateful/utxo_cohort.rs b/crates/brk_computer/src/stateful/utxo_cohort.rs index f837d2eda..a78f3302e 100644 --- a/crates/brk_computer/src/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/stateful/utxo_cohort.rs @@ -89,29 +89,27 @@ impl DynCohortVecs for Vecs { self.inner.validate_computed_versions(base_version) } - fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { + fn truncate_push(&mut self, height: Height) -> Result<()> { if self.state_starting_height.unwrap() > height { return Ok(()); } self.inner - .forced_pushed_at(height, exit, self.state.as_ref().unwrap()) + .truncate_push(height, self.state.as_ref().unwrap()) } - fn compute_then_force_push_unrealized_states( + fn compute_then_truncate_push_unrealized_states( &mut self, height: Height, height_price: Option, dateindex: Option, date_price: Option>, - exit: &Exit, ) -> Result<()> { - self.inner.compute_then_force_push_unrealized_states( + self.inner.compute_then_truncate_push_unrealized_states( height, height_price, dateindex, date_price, - exit, self.state.as_mut().unwrap(), ) } diff --git a/crates/brk_computer/src/traits.rs b/crates/brk_computer/src/traits.rs index 9d7d79c3c..5f2c09c86 100644 --- a/crates/brk_computer/src/traits.rs +++ b/crates/brk_computer/src/traits.rs @@ -71,7 +71,7 @@ impl ComputeDCAStackViaLen for EagerVec { prev = stack; - self.forced_push_at(i, stack, exit) + self.truncate_push_at(i, stack) })?; self.safe_flush(exit)?; @@ -115,7 +115,7 @@ impl ComputeDCAStackViaLen for EagerVec { prev = stack; - self.forced_push_at(i, stack, exit) + self.truncate_push_at(i, stack) })?; self.safe_flush(exit)?; @@ -173,7 +173,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec { .min(i.checked_sub(first_price_date).unwrap().to_usize() + 1) / Bitcoin::from(stack); } - self.forced_push_at(i, avg_price, exit) + self.truncate_push_at(i, avg_price) })?; self.safe_flush(exit)?; @@ -205,7 +205,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec { if i >= from { avg_price = DCA_AMOUNT * (i.to_usize() + 1 - from) / Bitcoin::from(stack); } - self.forced_push_at(i, avg_price, exit) + self.truncate_push_at(i, avg_price) })?; self.safe_flush(exit)?; diff --git a/crates/brk_indexer/examples/indexer_read.rs b/crates/brk_indexer/examples/indexer_read.rs index 07906c311..3bdda7ad3 100644 --- a/crates/brk_indexer/examples/indexer_read.rs +++ b/crates/brk_indexer/examples/indexer_read.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Sats; +// use brk_types::Sats; use std::{fs, path::Path}; fn main() -> Result<()> { diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index d687b5609..ac0f150ed 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -17,13 +17,13 @@ use rayon::prelude::*; use rustc_hash::{FxHashMap, FxHashSet}; use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, TypedVecIterator}; mod indexes; -// mod stores_v2; -mod stores_v3; +mod stores_v2; +// mod stores_v3; mod vecs; pub use indexes::*; -// pub use stores_v2::*; -pub use stores_v3::*; +pub use stores_v2::*; +// pub use stores_v3::*; pub use vecs::*; // One version for all data sources diff --git a/crates/brk_indexer/src/stores_v2.rs b/crates/brk_indexer/src/stores_v2.rs index f8b71fa6c..3fa2771dc 100644 --- a/crates/brk_indexer/src/stores_v2.rs +++ b/crates/brk_indexer/src/stores_v2.rs @@ -181,7 +181,7 @@ impl Stores { self.keyspace.inner().batch().commit_partitions(tuples)?; self.keyspace - .persist(PersistMode::SyncAll) + .persist(PersistMode::SyncData) .map_err(|e| e.into()) } diff --git a/crates/brk_indexer/src/stores_v3.rs b/crates/brk_indexer/src/stores_v3.rs index 6fd7575a8..1e4b45334 100644 --- a/crates/brk_indexer/src/stores_v3.rs +++ b/crates/brk_indexer/src/stores_v3.rs @@ -1,4 +1,4 @@ -use std::{fs, path::Path}; +use std::{fs, path::Path, time::Instant}; use brk_error::Result; use brk_grouper::ByAddressType; @@ -8,7 +8,8 @@ use brk_types::{ OutPoint, OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout, }; -use fjall3::{Database, PersistMode}; +use fjall3::{AbstractTree, Database, PersistMode}; +use log::info; use rayon::prelude::*; use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator, VecIndex, VecIterator}; @@ -18,7 +19,7 @@ use super::Vecs; #[derive(Clone)] pub struct Stores { - pub database: Database, + pub db: Database, pub addresstype_to_addresshash_to_addressindex: ByAddressType>, pub addresstype_to_addressindex_and_txindex: ByAddressType>, @@ -84,7 +85,7 @@ impl Stores { }; Ok(Self { - database: database.clone(), + db: database.clone(), height_to_coinbase_tag: Store::import( database_ref, @@ -103,14 +104,6 @@ impl Stores { addresstype_to_addressindex_and_unspentoutpoint: ByAddressType::new_with_index( create_addressindex_to_unspentoutpoint_store, )?, - // addresshash_to_typeindex: Store::import( - // database_ref, - // path, - // "addresshash_to_typeindex", - // version, - // Mode3::PushOnly, - // Kind3::Random, - // )?, blockhashprefix_to_height: Store::import( database_ref, path, @@ -127,22 +120,6 @@ impl Stores { Mode3::PushOnly, Kind3::Random, )?, - // addresstype_to_addressindex_and_txindex: Store::import( - // database_ref, - // path, - // "addresstype_to_addressindex_and_txindex", - // version, - // Mode3::PushOnly, - // Kind3::Vec, - // )?, - // addresstype_to_addressindex_and_unspentoutpoint: Store::import( - // database_ref, - // path, - // "addresstype_to_addressindex_and_unspentoutpoint", - // version, - // Mode3::Any, - // Kind3::Vec, - // )?, }) } @@ -158,6 +135,12 @@ impl Stores { } pub fn commit(&mut self, height: Height) -> Result<()> { + info!( + "self.db.config.cache.size = {}", + self.db.config.cache.size() + ); + + let i = Instant::now(); let tuples = [ &mut self.blockhashprefix_to_height as &mut dyn AnyStore, &mut self.height_to_coinbase_tag, @@ -185,12 +168,32 @@ impl Stores { Ok((store.keyspace(), items)) }) .collect::>>()?; + info!("Store items collected in {:?}", i.elapsed()); - self.database.batch().commit_keyspaces(tuples)?; + let version_memtable_size_sum = tuples + .iter() + .map(|(keyspace, _)| keyspace.tree.version_memtable_size_sum()) + .collect::>(); + // let sum = version_memtable_size_sum.iter().sum::(); + println!( + "version_memtable_size_sum = {:?} = ", + version_memtable_size_sum + ); - self.database - .persist(PersistMode::SyncAll) - .map_err(|e| e.into()) + let i = Instant::now(); + self.db.batch().commit_keyspaces(tuples)?; + info!("Batch done in {:?}", i.elapsed()); + + let i = Instant::now(); + self.db.persist(PersistMode::SyncData)?; + info!("Stores persisted in {:?}", i.elapsed()); + + info!( + "self.db.config.cache.size = {}", + self.db.config.cache.size() + ); + + Ok(()) } fn iter_any_store(&self) -> impl Iterator { diff --git a/crates/brk_store/src/fjall_v2/mod.rs b/crates/brk_store/src/fjall_v2/mod.rs index b39a1fb2a..5af5f7ecb 100644 --- a/crates/brk_store/src/fjall_v2/mod.rs +++ b/crates/brk_store/src/fjall_v2/mod.rs @@ -30,6 +30,7 @@ const MAJOR_FJALL_VERSION: Version = Version::TWO; pub fn open_keyspace(path: &Path) -> fjall2::Result { fjall2::Config::new(path.join("fjall")) .manual_journal_persist(true) + .max_write_buffer_size(256 * 1_024 * 1_024) .open_transactional() } diff --git a/crates/brk_store/src/fjall_v3/mod.rs b/crates/brk_store/src/fjall_v3/mod.rs index 4e4d33427..f0401cb63 100644 --- a/crates/brk_store/src/fjall_v3/mod.rs +++ b/crates/brk_store/src/fjall_v3/mod.rs @@ -28,6 +28,7 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3); pub fn open_fjall3_database(path: &Path) -> fjall3::Result { Database::builder(path.join("fjall")) + .max_write_buffer_size(256 * 1_024 * 1_024) .cache_size(4 * 1024 * 1024 * 1024) .open() } diff --git a/crates/brk_types/src/sats.rs b/crates/brk_types/src/sats.rs index 426847982..55add6694 100644 --- a/crates/brk_types/src/sats.rs +++ b/crates/brk_types/src/sats.rs @@ -197,6 +197,13 @@ impl Div for Sats { } } +impl From for Sats { + #[inline] + fn from(value: u8) -> Self { + Self(value as u64) + } +} + impl From for Sats { #[inline] fn from(value: u64) -> Self { diff --git a/crates/brk_types/src/stored_f32.rs b/crates/brk_types/src/stored_f32.rs index b0be39bd8..2a0f8ccb0 100644 --- a/crates/brk_types/src/stored_f32.rs +++ b/crates/brk_types/src/stored_f32.rs @@ -72,6 +72,13 @@ impl From for StoredF32 { } } +impl From for StoredF32 { + #[inline] + fn from(value: u8) -> Self { + Self(value as f32) + } +} + impl From for StoredF32 { #[inline] fn from(value: StoredU32) -> Self { diff --git a/crates/brk_types/src/stored_f64.rs b/crates/brk_types/src/stored_f64.rs index d80af3f84..e480c0f8a 100644 --- a/crates/brk_types/src/stored_f64.rs +++ b/crates/brk_types/src/stored_f64.rs @@ -2,7 +2,7 @@ use std::{ cmp::Ordering, f64, iter::Sum, - ops::{Add, AddAssign, Div, Mul}, + ops::{Add, AddAssign, Div, Mul, Sub}, }; use derive_deref::Deref; @@ -45,6 +45,13 @@ impl From for StoredF64 { } } +impl From for StoredF64 { + #[inline] + fn from(value: u8) -> Self { + Self(value as f64) + } +} + impl From for StoredF64 { #[inline] fn from(value: usize) -> Self { @@ -65,7 +72,14 @@ impl Mul for StoredF64 { } } -impl Mul for StoredF64 { +impl Sub for StoredF64 { + type Output = Self; + fn sub(self, rhs: Self) -> Self::Output { + Self(self.0 - rhs.0) + } +} + +impl Mul for StoredF64 { type Output = Self; fn mul(self, rhs: Self) -> Self::Output { Self(self.0 * rhs.0) @@ -86,7 +100,7 @@ impl Div for StoredF64 { } } -impl Div for StoredF64 { +impl Div for StoredF64 { type Output = Self; fn div(self, rhs: Self) -> Self::Output { Self(self.0 / rhs.0) diff --git a/docs/TODO.md b/docs/TODO.md index 13e795b29..869a9db2b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -150,3 +150,4 @@ - rename `output` to `txout` or `vout`, `input` to `txin` or `vin` - https://マリウス.com/thoughts-on-cloudflare/ - tor: https://news.ycombinator.com/item?id=45683380 + - design: https://opencode.ai/ diff --git a/websites/bitview/index.html b/websites/bitview/index.html index f09b310e7..31f13f3f7 100644 --- a/websites/bitview/index.html +++ b/websites/bitview/index.html @@ -222,21 +222,21 @@ --black: oklch(15% 0.005 44); --red: oklch(0.607 0.241 26.328); --orange: oklch(67.64% 0.191 44.41); - --amber: oklch(0.7175 0.1835 64.199); + --amber: oklch(0.7175 0.1835 64.199); /* #f08500 */ --yellow: oklch(0.738 0.173 80.9405); --avocado: oklch(0.72 0.19 110); - --lime: oklch(0.708 0.2165 131.267); + --lime: oklch(0.708 0.2165 131.267); /* #6dba00 */ --green: oklch(0.675 0.2065 149.3965); --emerald: oklch(0.646 0.1575 162.8525); --teal: oklch(0.652 0.129 183.6035); - --cyan: oklch(0.662 0.1345 218.472); + --cyan: oklch(0.662 0.1345 218.472); /* #00a5ca */ --sky: oklch(0.6365 0.1635 239.6445); --blue: oklch(0.5845 0.2295 261.348); --indigo: oklch(0.548 0.2475 277.0415); - --violet: oklch(0.5735 0.2655 292.863); - --purple: oklch(0.5925 0.2765 303.1105); + --violet: oklch(0.5735 0.2655 292.863); /* #863dff */ + --purple: oklch(0.5925 0.2765 303.1105); /* #a232ff */ --fuchsia: oklch(0.629 0.294 322.523); - --pink: oklch(0.624 0.245 357.444); + --pink: oklch(0.624 0.245 357.444); /* #ee1988 */ --rose: oklch(0.6155 0.2495 17.012); --background-color: light-dark(var(--white), var(--black)); --color: light-dark(var(--black), var(--white));