diff --git a/Cargo.lock b/Cargo.lock index f1947b5ab..ca305fac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,6 +415,7 @@ version = "0.0.8" dependencies = [ "brk_core", "brk_exit", + "brk_fetcher", "brk_indexer", "brk_logger", "brk_parser", @@ -550,6 +551,7 @@ dependencies = [ name = "brk_vec" version = "0.0.8" dependencies = [ + "brk_core", "brk_exit", "memmap2", "rayon", @@ -1478,10 +1480,11 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3590fea8e9e22d449600c9bbd481a8163bef223e4ff938e5f55899f8cf1adb93" +checksum = "5c163c633eb184a4ad2a5e7a5dacf12a58c830d717a7963563d4eceb4ced079f" dependencies = [ + "jiff-static", "jiff-tzdb-platform", "log", "portable-atomic", @@ -1491,10 +1494,21 @@ dependencies = [ ] [[package]] -name = "jiff-tzdb" -version = "0.1.2" +name = "jiff-static" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2cec2f5d266af45a071ece48b1fb89f3b00b2421ac3a5fe10285a6caaa60d3" +checksum = "dbc3e0019b0f5f43038cf46471b1312136f29e36f54436c6042c8f155fec8789" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962e1dfe9b2d75a84536cf5bf5eaaa4319aa7906c7160134a22883ac316d5f31" [[package]] name = "jiff-tzdb-platform" @@ -1774,9 +1788,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "oxc" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b4a5aba9c3f445db5401f9975266b23a5c5b49a8b4ccb1efa049132e96069dc" +checksum = "cae6276febbc5abc1f1e4cf49167d54ab341818656ae4f622d5992b65fcdd371" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1817,9 +1831,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7844c292ad3f79021bf3066538129c662a65bb4235a3450d124edd81bc9094df" +checksum = "22cc5cd078806a1b7061fa146dc4228a57d0765da6c85e99500d069b86f57e94" dependencies = [ "allocator-api2", "bumpalo", @@ -1830,9 +1844,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4005b2e219aa87cb0cbde495bcd6543748a4d77c8583ec573c6db2984bb249f" +checksum = "e4722414ac21a2e28a16b76de8390672c01a39adcb703d405b848149cfaeeaf7" dependencies = [ "bitflags", "cow-utils", @@ -1847,9 +1861,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93405fb1ad62247fa9296ad73265671b9c4aaa39dbbe142af00a6de415e61606" +checksum = "8f1505d8622b2ea6ed0274f355bd5e4ee3f09df5d9b39c8a3a673f344d87b82a" dependencies = [ "proc-macro2", "quote", @@ -1858,9 +1872,9 @@ dependencies = [ [[package]] name = "oxc_ast_visit" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f333b2ae5cbf1328986ce1eca317bc7f2f2e256c3d4a0f72b0d935a30c86b641" +checksum = "cf45370c6da0dd142a70468e5b25127d6a34caa71056105c85087559c8ee9afb" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1870,9 +1884,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775eee138628ad50d1d8f36bc7985659aed2f7457879ad3f493e1d705338d830" +checksum = "96be30717d29eb7d1780758d033e92fcc208b8cce83b3b4869d2155fa4c9b7bd" dependencies = [ "bitflags", "itertools", @@ -1885,9 +1899,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b7b756b328a516af07e3d86d82c6431ca21c4ab8abd626c13cec4bb478c564" +checksum = "e5fd891e06aa19c7d22e129d366e55644ffeed99167e3495dcaec8149a0631b3" dependencies = [ "bitflags", "cow-utils", @@ -1906,9 +1920,9 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc814bebc24ca429f257fba498dee171a98fa2ef1ea0f9c9eb3d69f0993baa6" +checksum = "cb49a2ee880952c2079b61851ecc35d7671d9d3509e306f5e704ccacd2783984" dependencies = [ "assert-unchecked", "ropey", @@ -1916,9 +1930,9 @@ dependencies = [ [[package]] name = "oxc_diagnostics" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd1f43ffbdabf86a563b48970c2abdbc0ac860e49558c83ab23a010b1a774f8" +checksum = "ea83fe2415b0580980ac83364c1ae943f8ee9c00becf5395a89e800a9526a080" dependencies = [ "cow-utils", "oxc-miette", @@ -1926,9 +1940,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a23d0e112ff9ea0f39dd2c9c921f0938e313e26ee9e4022f60c8d81c561e518" +checksum = "6c7e7bcc382cf901e93a16f86e70f2737c507aaa833656b0d8484d64f8ae358a" dependencies = [ "cow-utils", "num-bigint", @@ -1940,9 +1954,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd9818707125125e32d25fd18c5331a4418fbde084e33c69dee123df5b2531d2" +checksum = "1cb66484735d21f096b07c894badc96d89a0c6d31b4bdd46b33b3e44da9b97ac" [[package]] name = "oxc_index" @@ -1952,9 +1966,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d859623b93adfde62e5df257767c30f0f7d8b9427f9e3f94be695cf72d294e88" +checksum = "8069aaf088ccaee383e008ff29d407c2c88b4ff8694f7e34a9d6aaf202c81e71" dependencies = [ "fixedbitset", "itertools", @@ -1969,9 +1983,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba050a42f842dcf0628d46d1e817ee819661bd5283165b1784e924153912c32" +checksum = "1db16b19f75f29b9502b651c6ffae09366d43fd91decf5535d80c634f8e7e21d" dependencies = [ "cow-utils", "oxc_allocator", @@ -1991,9 +2005,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cce20364085569f8caebeb879aec455c5c8f6e5dcb5d0ecfb88208ae5e709331" +checksum = "7a7414e779b9723b0bd2880fe69b0ee517e583f88c5118a7c9053cf3317b95b1" dependencies = [ "assert-unchecked", "bitflags", @@ -2014,9 +2028,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e1f1e21ed9b3e4d1c6eb10dffc1c71202e8fe111968cf6636f5484ad443b0b" +checksum = "d18196c212eac24a7faf613e373e5f9317d3542578a088d01dc1a548fa1e1cb3" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -2030,9 +2044,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6997bba036cadea531b817c440d87b1b489fca6db981a3953a98b7b23201a13" +checksum = "4556f1c700baaec8589872ddf2af41d9a964db52fdbec9a5d07d7477dce45cf8" dependencies = [ "assert-unchecked", "itertools", @@ -2067,9 +2081,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e59da8e765f44892357f6b75bdb16f486d61956c5bd92a345429ad704d4853fd" +checksum = "318f925e26bd118adc082d290538d07611fe2434987a5c60cf5084381ecb42e6" dependencies = [ "compact_str", "oxc-miette", @@ -2080,9 +2094,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b8bd67fdd8df27cfddbec577b282f1a6601509ecd750697f590d7cdc2d88b7" +checksum = "8cb68ceb7c6902f3043fe8fe49bb886826b1d7741dc2904337297d53692b1b9c" dependencies = [ "assert-unchecked", "bitflags", @@ -2101,9 +2115,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.56.0" +version = "0.56.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3b19901f8ad9760deb01757c062d059c8632f7ca8b869c06f2ca4c96bfcba10" +checksum = "d8a090c9cd461d468f5faf3fe3f56378a4e27d80febe34ad39f2a04920f594d4" dependencies = [ "compact_str", "itoa", @@ -2432,9 +2446,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" -version = "0.17.12" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9b823fa29b721a59671b41d6b06e66b29e0628e207e8b1c3ceeda701ec928d" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", "cfg-if", @@ -2477,9 +2491,9 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8dcd64f141950290e45c99f7710ede1b600297c91818bb30b3667c0f45dc0" +checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657" dependencies = [ "bitflags", "errno", @@ -2943,9 +2957,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.43.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "9975ea0f48b5aa3972bf2d888c238182458437cc2a19374b81b25cdf1023fb3a" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 6e9735337..1085c39cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -resolver = "2" +resolver = "3" members = ["crates/*"] package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" @@ -26,7 +26,7 @@ clap = { version = "4.5.31", features = ["derive", "string"] } color-eyre = "0.6.3" derive_deref = "1.1.1" fjall = "2.6.7" -jiff = "0.2.1" +jiff = "0.2.3" log = { version = "0.4.26" } minreq = { version = "2.13.2", features = ["https", "serde_json"] } rayon = "1.10.0" diff --git a/README.md b/README.md index 4f7795bda..8b4b92907 100644 --- a/README.md +++ b/README.md @@ -53,13 +53,11 @@ Pricing: `0.01 BTC / month` *or* `0.1 BTC / year` ## Donate -image - -[bc1q950q4ukpxxm6wjjkv6cpq8jzpazaxrrwftctkt](bitcoin:bc1q950q4ukpxxm6wjjkv6cpq8jzpazaxrrwftctkt) +[`bc1q09 8zsm89 m7kgyz e338vf ejhpdt 92ua9p 3peuve`](bitcoin:bc1q098zsm89m7kgyze338vfejhpdt92ua9p3peuve) image -[lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4](lightning:lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4) +[`lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4`](lightning:lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4) [Geyser Fund](https://geyser.fund/project/brk) diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 4522c51eb..f34dbb718 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true [dependencies] brk_core = { workspace = true } brk_exit = { workspace = true } +brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index 2d50a500d..ad668b358 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -1,6 +1,7 @@ use std::{path::Path, thread::sleep, time::Duration}; use brk_computer::Computer; +use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::{ @@ -14,7 +15,8 @@ pub fn main() -> color_eyre::Result<()> { brk_logger::init(Some(Path::new(".log"))); - let bitcoin_dir = Path::new("../../../bitcoin"); + let bitcoin_dir = default_bitcoin_path(); + let rpc = Box::leak(Box::new(rpc::Client::new( "http://localhost:8332", rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 9f328dd0b..f4e4852ac 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -11,7 +11,6 @@ pub use brk_parser::rpc; mod storage; -use brk_core::Date; use log::info; use storage::{Stores, Vecs}; @@ -53,101 +52,7 @@ impl Computer { ) -> color_eyre::Result<()> { info!("Computing..."); - let height_count = indexer.vecs().height_to_size.len(); - let txindexes_count = indexer.vecs().txindex_to_txid.len(); - let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len(); - let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len(); - - // self.vecs.txindex_to_last_txinindex.compute_last_index_from_first( - // starting_indexes.txindex, - // &mut indexer.vecs().txindex_to_first_txinindex, - // txinindexes_count, - // exit, - // )?; - - // self.vecs.txindex_to_inputs_count.compute_count_from_indexes( - // starting_indexes.txindex, - // &mut indexer.vecs().txindex_to_first_txinindex, - // &mut self.vecs.txindex_to_last_txinindex, - // exit, - // )?; - - // self.vecs.txindex_to_last_txoutindex.compute_last_index_from_first( - // starting_indexes.txindex, - // &mut indexer.vecs().txindex_to_first_txoutindex, - // txoutindexes_count, - // exit, - // )?; - - // self.vecs.txindex_to_outputs_count.compute_count_from_indexes( - // starting_indexes.txindex, - // &mut indexer.vecs().txindex_to_first_txoutindex, - // &mut self.vecs.txindex_to_last_txoutindex, - // exit, - // )?; - - self.mut_vecs().height_to_height.compute_transform( - starting_indexes.height, - &mut indexer.mut_vecs().height_to_timestamp, - |_, height| height, - exit, - )?; - - self.mut_vecs().height_to_date.compute_transform( - starting_indexes.height, - &mut indexer.mut_vecs().height_to_timestamp, - |timestamp, _| Date::from(*timestamp), - exit, - )?; - - // self.vecs.height_to_last_txindex.compute_last_index_from_first( - // starting_indexes.height, - // &mut indexer.vecs().height_to_first_txindex, - // height_count, - // exit, - // )?; - - // self.vecs.txindex_to_height.compute_inverse_less_to_more( - // starting_indexes.height, - // &mut indexer.vecs().height_to_first_txindex, - // &mut self.vecs.height_to_last_txindex, - // exit, - // )?; - - // self.vecs.txindex_to_is_coinbase.compute_is_first_ordered( - // starting_indexes.txindex, - // &mut self.vecs.txindex_to_height, - // &mut indexer.vecs().height_to_first_txindex, - // exit, - // )?; - - // self.vecs.txindex_to_fee.compute_transform( - // &mut self.vecs.txindex_to_height, - // &mut indexer.vecs().height_to_first_txindex, - // )?; - - let date_count = self.vecs().height_to_date.len(); - - // self.vecs.height_to_dateindex.compute(...) - - // self.vecs - // .dateindex_to_first_height - // .compute_inverse_more_to_less(&mut self.vecs.height_to_dateindex, exit)?; - - // --- - // Date to X - // --- - // ... - - // --- - // Month to X - // --- - // ... - - // --- - // Year to X - // --- - // ... + self.mut_vecs().compute(indexer, starting_indexes, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/storage/vecs.rs b/crates/brk_computer/src/storage/vecs.rs deleted file mode 100644 index ecb8ebe58..000000000 --- a/crates/brk_computer/src/storage/vecs.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::{fs, path::Path}; - -use brk_core::{ - Addressindex, Cents, Close, Date, Dateindex, Dollars, Feerate, Height, High, Low, Open, Sats, Timestamp, Txindex, - Txinindex, Txoutindex, -}; -use brk_vec::{AnyStorableVec, StorableVec, Version}; - -// mod base; - -// use base::*; - -#[derive(Clone)] -pub struct Vecs { - pub dateindex_to_first_height: StorableVec, - // pub dateindex_to_last_height: StorableVec, - // pub height_to_block_interval: StorableVec, - pub dateindex_to_close_in_cents: StorableVec>, - pub dateindex_to_close_in_dollars: StorableVec>, - pub dateindex_to_high_in_cents: StorableVec>, - pub dateindex_to_high_in_dollars: StorableVec>, - pub dateindex_to_low_in_cents: StorableVec>, - pub dateindex_to_low_in_dollars: StorableVec>, - pub dateindex_to_open_in_cents: StorableVec>, - pub dateindex_to_open_in_dollars: StorableVec>, - pub height_to_close_in_cents: StorableVec>, - pub height_to_close_in_dollars: StorableVec>, - pub height_to_height: StorableVec, - pub height_to_high_in_cents: StorableVec>, - pub height_to_high_in_dollars: StorableVec>, - pub height_to_low_in_cents: StorableVec>, - pub height_to_low_in_dollars: StorableVec>, - pub height_to_open_in_cents: StorableVec>, - pub height_to_open_in_dollars: StorableVec>, - pub height_to_date: StorableVec, - pub height_to_dateindex: StorableVec, - // pub height_to_fee: StorableVec, - // pub height_to_inputcount: StorableVec, - // pub height_to_last_addressindex: StorableVec, - pub height_to_last_txindex: StorableVec, - // pub height_to_last_txoutindex: StorableVec, - // pub height_to_maxfeerate: StorableVec, - // pub height_to_medianfeerate: StorableVec, - // pub height_to_minfeerate: StorableVec, - // pub height_to_outputcount: StorableVec, - // pub height_to_subsidy: StorableVec, - // pub height_to_totalfees: StorableVec, - // pub height_to_txcount: StorableVec, - pub txindex_to_fee: StorableVec, - pub txindex_to_height: StorableVec, - pub txindex_to_is_coinbase: StorableVec, - // pub txindex_to_feerate: StorableVec, - pub txindex_to_inputs_count: StorableVec, - pub txindex_to_inputs_sum: StorableVec, - pub txindex_to_last_txinindex: StorableVec, - pub txindex_to_last_txoutindex: StorableVec, - pub txindex_to_outputs_count: StorableVec, - pub txindex_to_outputs_sum: StorableVec, -} - -impl Vecs { - pub fn import(path: &Path) -> color_eyre::Result { - fs::create_dir_all(path)?; - - Ok(Self { - dateindex_to_first_height: StorableVec::forced_import( - &path.join("dateindex_to_first_height"), - Version::from(1), - )?, - // height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?, - dateindex_to_close_in_cents: StorableVec::import( - &path.join("dateindex_to_close_in_cents"), - Version::from(1), - )?, - dateindex_to_close_in_dollars: StorableVec::import( - &path.join("dateindex_to_close_in_dollars"), - Version::from(1), - )?, - dateindex_to_high_in_cents: StorableVec::import( - &path.join("dateindex_to_high_in_cents"), - Version::from(1), - )?, - dateindex_to_high_in_dollars: StorableVec::import( - &path.join("dateindex_to_high_in_dollars"), - Version::from(1), - )?, - dateindex_to_low_in_cents: StorableVec::import(&path.join("dateindex_to_low_in_cents"), Version::from(1))?, - dateindex_to_low_in_dollars: StorableVec::import( - &path.join("dateindex_to_low_in_dollars"), - Version::from(1), - )?, - dateindex_to_open_in_cents: StorableVec::import( - &path.join("dateindex_to_open_in_cents"), - Version::from(1), - )?, - dateindex_to_open_in_dollars: StorableVec::import( - &path.join("dateindex_to_open_in_dollars"), - Version::from(1), - )?, - height_to_close_in_cents: StorableVec::import(&path.join("height_to_close_in_cents"), Version::from(1))?, - height_to_close_in_dollars: StorableVec::import( - &path.join("height_to_close_in_dollars"), - Version::from(1), - )?, - height_to_height: StorableVec::import(&path.join("height_to_height"), Version::from(1))?, - height_to_high_in_cents: StorableVec::import(&path.join("height_to_high_in_cents"), Version::from(1))?, - height_to_high_in_dollars: StorableVec::import(&path.join("height_to_high_in_dollars"), Version::from(1))?, - height_to_low_in_cents: StorableVec::import(&path.join("height_to_low_in_cents"), Version::from(1))?, - height_to_low_in_dollars: StorableVec::import(&path.join("height_to_low_in_dollars"), Version::from(1))?, - height_to_open_in_cents: StorableVec::import(&path.join("height_to_open_in_cents"), Version::from(1))?, - height_to_open_in_dollars: StorableVec::import(&path.join("height_to_open_in_dollars"), Version::from(1))?, - height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?, - height_to_dateindex: StorableVec::forced_import(&path.join("height_to_dateindex"), Version::from(1))?, - // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?, - // height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?, - // height_to_last_addressindex: StorableVec::forced_import( - // &path.join("height_to_last_addressindex"), - // Version::from(1), - // )?, - height_to_last_txindex: StorableVec::forced_import(&path.join("height_to_last_txindex"), Version::from(1))?, - // height_to_last_txoutindex: StorableVec::forced_import(&path.join("height_to_last_txoutindex"), Version::from(1))?, - // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?, - // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?, - // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?, - // height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?, - // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?, - // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?, - // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?, - txindex_to_fee: StorableVec::forced_import(&path.join("txindex_to_fee"), Version::from(1))?, - txindex_to_height: StorableVec::forced_import(&path.join("txindex_to_height"), Version::from(1))?, - txindex_to_is_coinbase: StorableVec::forced_import(&path.join("txindex_to_is_coinbase"), Version::from(1))?, - // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?, - txindex_to_inputs_count: StorableVec::forced_import( - &path.join("txindex_to_inputs_count"), - Version::from(1), - )?, - txindex_to_inputs_sum: StorableVec::forced_import(&path.join("txindex_to_inputs_sum"), Version::from(1))?, - txindex_to_last_txinindex: StorableVec::forced_import( - &path.join("txindex_to_last_txinindex"), - Version::from(1), - )?, - txindex_to_last_txoutindex: StorableVec::forced_import( - &path.join("txindex_to_last_txoutindex"), - Version::from(1), - )?, - txindex_to_outputs_count: StorableVec::forced_import( - &path.join("txindex_to_outputs_count"), - Version::from(1), - )?, - txindex_to_outputs_sum: StorableVec::forced_import(&path.join("txindex_to_outputs_sum"), Version::from(1))?, - }) - } - - pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { - vec![ - &self.height_to_date as &dyn AnyStorableVec, - &self.height_to_height, - // &self.dateindex_to_close_in_dollars, - // &self.dateindex_to_high_in_cents, - // &self.dateindex_to_high_in_dollars, - // &self.dateindex_to_low_in_cents, - // &self.dateindex_to_low_in_dollars, - // &self.dateindex_to_open_in_cents, - // &self.dateindex_to_open_in_dollars, - // &self.height_to_close_in_cents, - // &self.height_to_close_in_dollars, - // &self.height_to_high_in_cents, - // &self.height_to_high_in_dollars, - // &self.height_to_low_in_cents, - // &self.height_to_low_in_dollars, - // &self.height_to_open_in_cents, - // &self.height_to_open_in_dollars, - ] - } -} diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs new file mode 100644 index 000000000..ef5f8d4cc --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -0,0 +1,246 @@ +use std::{fs, ops::Deref, path::Path}; + +use brk_core::{Date, Dateindex, Height}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyStorableVec, StorableVec, Value, Version}; + +#[derive(Clone)] +pub struct Vecs { + pub dateindex_to_date: StorableVec, + pub dateindex_to_dateindex: StorableVec, + pub dateindex_to_first_height: StorableVec, + pub dateindex_to_last_height: StorableVec, + pub height_to_real_date: StorableVec, + pub height_to_fixed_date: StorableVec, + pub height_to_height: StorableVec, + pub height_to_dateindex: StorableVec, +} + +impl Vecs { + pub fn import(path: &Path) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + dateindex_to_date: StorableVec::forced_import( + &path.join("dateindex_to_date"), + Version::from(1), + )?, + dateindex_to_dateindex: StorableVec::forced_import( + &path.join("dateindex_to_dateindex"), + Version::from(1), + )?, + dateindex_to_first_height: StorableVec::forced_import( + &path.join("dateindex_to_first_height"), + Version::from(1), + )?, + dateindex_to_last_height: StorableVec::forced_import( + &path.join("dateindex_to_last_height"), + Version::from(1), + )?, + height_to_real_date: StorableVec::forced_import( + &path.join("height_to_real_date"), + Version::from(1), + )?, + height_to_fixed_date: StorableVec::forced_import( + &path.join("height_to_fixed_date"), + Version::from(1), + )?, + height_to_dateindex: StorableVec::forced_import( + &path.join("height_to_dateindex"), + Version::from(1), + )?, + height_to_height: StorableVec::forced_import( + &path.join("height_to_height"), + Version::from(1), + )?, + }) + } + + pub fn compute( + &mut self, + indexer: &mut Indexer, + starting_indexes: brk_indexer::Indexes, + exit: &Exit, + ) -> color_eyre::Result { + self.height_to_height.compute_transform( + starting_indexes.height, + &mut indexer.mut_vecs().height_to_timestamp, + |(h, ..)| (h, h), + exit, + )?; + + self.height_to_real_date.compute_transform( + starting_indexes.height, + &mut indexer.mut_vecs().height_to_timestamp, + |(h, t, ..)| (h, Date::from(*t)), + exit, + )?; + + self.height_to_fixed_date.compute_transform( + starting_indexes.height, + &mut self.height_to_real_date, + |(h, d, s, ..)| { + let d = h + .decremented() + .and_then(|h| s.read(h).ok()) + .flatten() + .map_or(*d, |prev_d| if prev_d > d { *prev_d } else { *d }); + (h, d) + }, + exit, + )?; + + self.height_to_dateindex.compute_transform( + starting_indexes.height, + &mut self.height_to_fixed_date, + |(h, d, ..)| (h, Dateindex::try_from(*d).unwrap()), + exit, + )?; + + let starting_dateindex = self + .height_to_dateindex + .get(starting_indexes.height.decremented().unwrap_or_default())? + .map(Value::into_inner) + .unwrap_or_default(); + + self.dateindex_to_first_height + .compute_inverse_more_to_less( + starting_indexes.height, + &mut self.height_to_dateindex, + exit, + )?; + + let date_len = self.dateindex_to_first_height.len(); + self.dateindex_to_last_height + .compute_last_index_from_first( + starting_dateindex, + &mut self.dateindex_to_first_height, + date_len, + exit, + )?; + + self.dateindex_to_dateindex.compute_transform( + starting_dateindex, + &mut self.dateindex_to_first_height, + |(di, ..)| (di, di), + exit, + )?; + + self.dateindex_to_date.compute_transform( + starting_dateindex, + &mut self.dateindex_to_dateindex, + |(di, ..)| (di, Date::from(di)), + exit, + )?; + + // let height_count = indexer.vecs().height_to_size.len(); + // let txindexes_count = indexer.vecs().txindex_to_txid.len(); + // let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len(); + // let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len(); + // let date_count = self.vecs().height_to_date.len(); + + // self.vecs.txindex_to_last_txinindex.compute_last_index_from_first( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txinindex, + // txinindexes_count, + // exit, + // )?; + + // self.vecs.txindex_to_inputs_count.compute_count_from_indexes( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txinindex, + // &mut self.vecs.txindex_to_last_txinindex, + // exit, + // )?; + + // self.vecs.txindex_to_last_txoutindex.compute_last_index_from_first( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txoutindex, + // txoutindexes_count, + // exit, + // )?; + + // self.vecs.txindex_to_outputs_count.compute_count_from_indexes( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txoutindex, + // &mut self.vecs.txindex_to_last_txoutindex, + // exit, + // )?; + + // self.vecs.height_to_last_txindex.compute_last_index_from_first( + // starting_indexes.height, + // &mut indexer.vecs().height_to_first_txindex, + // height_count, + // exit, + // )?; + + // self.vecs.txindex_to_height.compute_inverse_less_to_more( + // starting_indexes.height, + // &mut indexer.vecs().height_to_first_txindex, + // &mut self.vecs.height_to_last_txindex, + // exit, + // )?; + + // self.vecs.txindex_to_is_coinbase.compute_is_first_ordered( + // starting_indexes.txindex, + // &mut self.vecs.txindex_to_height, + // &mut indexer.vecs().height_to_first_txindex, + // exit, + // )?; + + // self.vecs.txindex_to_fee.compute_transform( + // &mut self.vecs.txindex_to_height, + // &mut indexer.vecs().height_to_first_txindex, + // )?; + + // self.vecs.height_to_dateindex.compute(...) + + // --- + // Date to X + // --- + // ... + + // --- + // Month to X + // --- + // ... + + // --- + // Year to X + // --- + // ... + + Ok(Indexes::from((starting_indexes, starting_dateindex))) + } + + pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + vec![ + &self.dateindex_to_date as &dyn AnyStorableVec, + &self.dateindex_to_dateindex, + &self.dateindex_to_first_height, + &self.dateindex_to_last_height, + &self.height_to_real_date, + &self.height_to_fixed_date, + &self.height_to_height, + &self.height_to_dateindex, + ] + } +} + +pub struct Indexes { + indexes: brk_indexer::Indexes, + pub dateindex: Dateindex, +} + +impl Deref for Indexes { + type Target = brk_indexer::Indexes; + fn deref(&self) -> &Self::Target { + &self.indexes + } +} +impl From<(brk_indexer::Indexes, Dateindex)> for Indexes { + fn from((indexes, dateindex): (brk_indexer::Indexes, Dateindex)) -> Self { + Self { indexes, dateindex } + } +} diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs new file mode 100644 index 000000000..845e1b631 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -0,0 +1,302 @@ +use std::{fs, path::Path}; + +use brk_core::{Cents, Close, Dateindex, Dollars, Height, High, Low, OHLCCents, OHLCDollars, Open}; +use brk_exit::Exit; +use brk_fetcher::Fetcher; +use brk_indexer::Indexer; +use brk_vec::{AnyStorableVec, StorableVec, Value, Version}; + +use super::indexes::{self, Indexes}; + +#[derive(Clone)] +pub struct Vecs { + pub dateindex_to_ohlc_in_cents: StorableVec, + pub dateindex_to_ohlc: StorableVec, + pub dateindex_to_close_in_cents: StorableVec>, + pub dateindex_to_close: StorableVec>, + pub dateindex_to_high_in_cents: StorableVec>, + pub dateindex_to_high: StorableVec>, + pub dateindex_to_low_in_cents: StorableVec>, + pub dateindex_to_low: StorableVec>, + pub dateindex_to_open_in_cents: StorableVec>, + pub dateindex_to_open: StorableVec>, + pub height_to_ohlc_in_cents: StorableVec, + pub height_to_ohlc: StorableVec, + pub height_to_close_in_cents: StorableVec>, + pub height_to_close: StorableVec>, + pub height_to_high_in_cents: StorableVec>, + pub height_to_high: StorableVec>, + pub height_to_low_in_cents: StorableVec>, + pub height_to_low: StorableVec>, + pub height_to_open_in_cents: StorableVec>, + pub height_to_open: StorableVec>, +} + +impl Vecs { + pub fn import(path: &Path) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + dateindex_to_ohlc_in_cents: StorableVec::import( + &path.join("dateindex_to_ohlc_in_cents"), + Version::from(1), + )?, + dateindex_to_ohlc: StorableVec::import( + &path.join("dateindex_to_ohlc"), + Version::from(1), + )?, + dateindex_to_close_in_cents: StorableVec::import( + &path.join("dateindex_to_close_in_cents"), + Version::from(1), + )?, + dateindex_to_close: StorableVec::import( + &path.join("dateindex_to_close"), + Version::from(1), + )?, + dateindex_to_high_in_cents: StorableVec::import( + &path.join("dateindex_to_high_in_cents"), + Version::from(1), + )?, + dateindex_to_high: StorableVec::import( + &path.join("dateindex_to_high"), + Version::from(1), + )?, + dateindex_to_low_in_cents: StorableVec::import( + &path.join("dateindex_to_low_in_cents"), + Version::from(1), + )?, + dateindex_to_low: StorableVec::import( + &path.join("dateindex_to_low"), + Version::from(1), + )?, + dateindex_to_open_in_cents: StorableVec::import( + &path.join("dateindex_to_open_in_cents"), + Version::from(1), + )?, + dateindex_to_open: StorableVec::import( + &path.join("dateindex_to_open"), + Version::from(1), + )?, + height_to_ohlc_in_cents: StorableVec::import( + &path.join("height_to_ohlc_in_cents"), + Version::from(1), + )?, + height_to_ohlc: StorableVec::import(&path.join("height_to_ohlc"), Version::from(1))?, + height_to_close_in_cents: StorableVec::import( + &path.join("height_to_close_in_cents"), + Version::from(1), + )?, + height_to_close: StorableVec::import(&path.join("height_to_close"), Version::from(1))?, + height_to_high_in_cents: StorableVec::import( + &path.join("height_to_high_in_cents"), + Version::from(1), + )?, + height_to_high: StorableVec::import(&path.join("height_to_high"), Version::from(1))?, + height_to_low_in_cents: StorableVec::import( + &path.join("height_to_low_in_cents"), + Version::from(1), + )?, + height_to_low: StorableVec::import(&path.join("height_to_low"), Version::from(1))?, + height_to_open_in_cents: StorableVec::import( + &path.join("height_to_open_in_cents"), + Version::from(1), + )?, + height_to_open: StorableVec::import(&path.join("height_to_open"), Version::from(1))?, + }) + } + + pub fn compute( + &mut self, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + starting_indexes: Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> { + let mut fetcher = Fetcher::import(None)?; + + self.height_to_ohlc_in_cents.compute_transform( + starting_indexes.height, + &mut indexer.mut_vecs().height_to_timestamp, + |(h, t, _, height_to_timestamp)| { + let ohlc = fetcher + .get_height( + h, + *t, + h.decremented().map(|prev_h| { + height_to_timestamp + .get(prev_h) + .unwrap() + .map(Value::into_inner) + .unwrap() + }), + ) + .unwrap(); + (h, ohlc) + }, + exit, + )?; + + self.height_to_open_in_cents.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + + self.height_to_high_in_cents.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + + self.height_to_low_in_cents.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + + self.height_to_close_in_cents.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + + self.height_to_ohlc.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), + exit, + )?; + + self.height_to_open.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + + self.height_to_high.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + + self.height_to_low.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + + self.height_to_close.compute_transform( + starting_indexes.height, + &mut self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + + self.dateindex_to_ohlc_in_cents.compute_transform( + starting_indexes.dateindex, + &mut indexes.dateindex_to_date, + |(di, d, ..)| { + let ohlc = fetcher.get_date(*d).unwrap(); + (di, ohlc) + }, + exit, + )?; + + self.dateindex_to_open_in_cents.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + + self.dateindex_to_high_in_cents.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + + self.dateindex_to_low_in_cents.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + + self.dateindex_to_close_in_cents.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + + self.dateindex_to_ohlc.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), + exit, + )?; + + self.dateindex_to_open.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + + self.dateindex_to_high.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + + self.dateindex_to_low.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + + self.dateindex_to_close.compute_transform( + starting_indexes.dateindex, + &mut self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + + Ok(()) + } + + pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + vec![ + &self.dateindex_to_close as &dyn AnyStorableVec, + &self.dateindex_to_close_in_cents, + &self.dateindex_to_high, + &self.dateindex_to_high_in_cents, + &self.dateindex_to_low, + &self.dateindex_to_low_in_cents, + &self.dateindex_to_ohlc, + &self.dateindex_to_ohlc_in_cents, + &self.dateindex_to_open, + &self.dateindex_to_open_in_cents, + &self.height_to_close, + &self.height_to_close_in_cents, + &self.height_to_high, + &self.height_to_high_in_cents, + &self.height_to_low, + &self.height_to_low_in_cents, + &self.height_to_ohlc, + &self.height_to_ohlc_in_cents, + &self.height_to_open, + &self.height_to_open_in_cents, + ] + } +} diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs new file mode 100644 index 000000000..bcbdf4e28 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -0,0 +1,202 @@ +use std::{fs, path::Path}; + +use brk_core::{Height, Sats, Txindex, Txinindex, Txoutindex}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyStorableVec, StorableVec, Version}; + +mod indexes; +mod marketprice; + +#[derive(Clone)] +pub struct Vecs { + pub indexes: indexes::Vecs, + pub marketprice: marketprice::Vecs, + // pub height_to_block_interval: StorableVec, + // pub height_to_fee: StorableVec, + // pub height_to_inputcount: StorableVec, + // pub height_to_last_addressindex: StorableVec, + pub height_to_last_txindex: StorableVec, + // pub height_to_last_txoutindex: StorableVec, + // pub height_to_maxfeerate: StorableVec, + // pub height_to_medianfeerate: StorableVec, + // pub height_to_minfeerate: StorableVec, + // pub height_to_outputcount: StorableVec, + // pub height_to_subsidy: StorableVec, + // pub height_to_totalfees: StorableVec, + // pub height_to_txcount: StorableVec, + pub txindex_to_fee: StorableVec, + pub txindex_to_height: StorableVec, + pub txindex_to_is_coinbase: StorableVec, + // pub txindex_to_feerate: StorableVec, + pub txindex_to_inputs_count: StorableVec, + pub txindex_to_inputs_sum: StorableVec, + pub txindex_to_last_txinindex: StorableVec, + pub txindex_to_last_txoutindex: StorableVec, + pub txindex_to_outputs_count: StorableVec, + pub txindex_to_outputs_sum: StorableVec, +} + +impl Vecs { + pub fn import(path: &Path) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + // height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?, + indexes: indexes::Vecs::import(path)?, + marketprice: marketprice::Vecs::import(path)?, + // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?, + // height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?, + // height_to_last_addressindex: StorableVec::forced_import( + // &path.join("height_to_last_addressindex"), + // Version::from(1), + // )?, + height_to_last_txindex: StorableVec::forced_import( + &path.join("height_to_last_txindex"), + Version::from(1), + )?, + // height_to_last_txoutindex: StorableVec::forced_import(&path.join("height_to_last_txoutindex"), Version::from(1))?, + // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?, + // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?, + // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?, + // height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?, + // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?, + // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?, + // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?, + txindex_to_fee: StorableVec::forced_import( + &path.join("txindex_to_fee"), + Version::from(1), + )?, + txindex_to_height: StorableVec::forced_import( + &path.join("txindex_to_height"), + Version::from(1), + )?, + txindex_to_is_coinbase: StorableVec::forced_import( + &path.join("txindex_to_is_coinbase"), + Version::from(1), + )?, + // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?, + txindex_to_inputs_count: StorableVec::forced_import( + &path.join("txindex_to_inputs_count"), + Version::from(1), + )?, + txindex_to_inputs_sum: StorableVec::forced_import( + &path.join("txindex_to_inputs_sum"), + Version::from(1), + )?, + txindex_to_last_txinindex: StorableVec::forced_import( + &path.join("txindex_to_last_txinindex"), + Version::from(1), + )?, + txindex_to_last_txoutindex: StorableVec::forced_import( + &path.join("txindex_to_last_txoutindex"), + Version::from(1), + )?, + txindex_to_outputs_count: StorableVec::forced_import( + &path.join("txindex_to_outputs_count"), + Version::from(1), + )?, + txindex_to_outputs_sum: StorableVec::forced_import( + &path.join("txindex_to_outputs_sum"), + Version::from(1), + )?, + }) + } + + pub fn compute( + &mut self, + indexer: &mut Indexer, + starting_indexes: brk_indexer::Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> { + let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; + + self.marketprice + .compute(indexer, &mut self.indexes, starting_indexes, exit)?; + + // self.mut_vecs().height_to_ohlc + + // let height_count = indexer.vecs().height_to_size.len(); + // let txindexes_count = indexer.vecs().txindex_to_txid.len(); + // let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len(); + // let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len(); + // let date_count = self.vecs().height_to_date.len(); + + // self.vecs.txindex_to_last_txinindex.compute_last_index_from_first( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txinindex, + // txinindexes_count, + // exit, + // )?; + + // self.vecs.txindex_to_inputs_count.compute_count_from_indexes( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txinindex, + // &mut self.vecs.txindex_to_last_txinindex, + // exit, + // )?; + + // self.vecs.txindex_to_last_txoutindex.compute_last_index_from_first( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txoutindex, + // txoutindexes_count, + // exit, + // )?; + + // self.vecs.txindex_to_outputs_count.compute_count_from_indexes( + // starting_indexes.txindex, + // &mut indexer.vecs().txindex_to_first_txoutindex, + // &mut self.vecs.txindex_to_last_txoutindex, + // exit, + // )?; + + // self.vecs.height_to_last_txindex.compute_last_index_from_first( + // starting_indexes.height, + // &mut indexer.vecs().height_to_first_txindex, + // height_count, + // exit, + // )?; + + // self.vecs.txindex_to_height.compute_inverse_less_to_more( + // starting_indexes.height, + // &mut indexer.vecs().height_to_first_txindex, + // &mut self.vecs.height_to_last_txindex, + // exit, + // )?; + + // self.vecs.txindex_to_is_coinbase.compute_is_first_ordered( + // starting_indexes.txindex, + // &mut self.vecs.txindex_to_height, + // &mut indexer.vecs().height_to_first_txindex, + // exit, + // )?; + + // self.vecs.txindex_to_fee.compute_transform( + // &mut self.vecs.txindex_to_height, + // &mut indexer.vecs().height_to_first_txindex, + // )?; + + // self.vecs.height_to_dateindex.compute(...) + + // --- + // Date to X + // --- + // ... + + // --- + // Month to X + // --- + // ... + + // --- + // Year to X + // --- + // ... + + Ok(()) + } + + pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + [self.indexes.as_any_vecs(), self.marketprice.as_any_vecs()].concat() + } +} diff --git a/crates/brk_computer/src/storage/vecs_/base.rs b/crates/brk_computer/src/storage/vecs_/base.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/crates/brk_computer/src/storage/vecs_/mod.rs b/crates/brk_computer/src/storage/vecs_/mod.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/crates/brk_core/src/structs/addressindex.rs b/crates/brk_core/src/structs/addressindex.rs index 01977413f..74379139b 100644 --- a/crates/brk_core/src/structs/addressindex.rs +++ b/crates/brk_core/src/structs/addressindex.rs @@ -1,7 +1,6 @@ use std::ops::Add; use byteview::ByteView; -use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -15,8 +14,6 @@ use crate::Error; Ord, Clone, Copy, - Deref, - DerefMut, Default, FromBytes, Immutable, @@ -29,16 +26,12 @@ pub struct Addressindex(u32); impl Addressindex { pub const BYTES: usize = size_of::(); - pub fn decremented(self) -> Self { - Self(*self - 1) - } - pub fn increment(&mut self) { self.0 += 1; } pub fn incremented(self) -> Self { - Self(*self + 1) + Self(self.0 + 1) } } diff --git a/crates/brk_core/src/structs/addresstypeindex.rs b/crates/brk_core/src/structs/addresstypeindex.rs index 97f99c533..3bb31026e 100644 --- a/crates/brk_core/src/structs/addresstypeindex.rs +++ b/crates/brk_core/src/structs/addresstypeindex.rs @@ -12,8 +12,6 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; Ord, Clone, Copy, - Deref, - DerefMut, Default, FromBytes, Immutable, @@ -24,16 +22,12 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; pub struct Addresstypeindex(u32); impl Addresstypeindex { - pub fn decremented(self) -> Self { - Self(*self - 1) - } - pub fn increment(&mut self) { self.0 += 1; } pub fn incremented(self) -> Self { - Self(*self + 1) + Self(self.0 + 1) } pub fn copy_then_increment(&mut self) -> Self { diff --git a/crates/brk_core/src/structs/bitcoin.rs b/crates/brk_core/src/structs/bitcoin.rs index e7f9ef5fe..a56706a6b 100644 --- a/crates/brk_core/src/structs/bitcoin.rs +++ b/crates/brk_core/src/structs/bitcoin.rs @@ -9,6 +9,6 @@ impl Bitcoin { impl From for Bitcoin { fn from(value: Sats) -> Self { - Self((*value as f64) / Self::ONE.0) + Self(f64::from(value) / Self::ONE.0) } } diff --git a/crates/brk_core/src/structs/cents.rs b/crates/brk_core/src/structs/cents.rs index 6130b14f8..97fb9460a 100644 --- a/crates/brk_core/src/structs/cents.rs +++ b/crates/brk_core/src/structs/cents.rs @@ -1,4 +1,3 @@ -use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -13,7 +12,6 @@ use super::Dollars; Eq, PartialOrd, Ord, - Deref, FromBytes, Immutable, IntoBytes, @@ -27,3 +25,9 @@ impl From for Cents { Self((*value * 100.0).floor() as u64) } } + +impl From for f64 { + fn from(value: Cents) -> Self { + value.0 as f64 + } +} diff --git a/crates/brk_core/src/structs/date.rs b/crates/brk_core/src/structs/date.rs index 8aed1398d..1e8c3f81b 100644 --- a/crates/brk_core/src/structs/date.rs +++ b/crates/brk_core/src/structs/date.rs @@ -60,11 +60,15 @@ impl From for Date { impl From for Date { fn from(value: Dateindex) -> Self { - Self::from( - Self::INDEX_ZERO_ - .checked_add(Span::new().days(i64::from(value))) - .unwrap(), - ) + if value == Dateindex::default() { + Date::INDEX_ZERO + } else { + Self::from( + Self::INDEX_ONE_ + .checked_add(Span::new().days(i64::from(value) - 1)) + .unwrap(), + ) + } } } diff --git a/crates/brk_core/src/structs/dateindex.rs b/crates/brk_core/src/structs/dateindex.rs index ed4e92042..3322e6c17 100644 --- a/crates/brk_core/src/structs/dateindex.rs +++ b/crates/brk_core/src/structs/dateindex.rs @@ -9,10 +9,26 @@ use crate::Error; use super::Date; #[derive( - Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize, + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, )] pub struct Dateindex(u16); +impl Dateindex { + pub const BYTES: usize = size_of::(); +} + impl From for usize { fn from(value: Dateindex) -> Self { value.0 as usize diff --git a/crates/brk_core/src/structs/dollars.rs b/crates/brk_core/src/structs/dollars.rs index 4ea1892b3..5544457b2 100644 --- a/crates/brk_core/src/structs/dollars.rs +++ b/crates/brk_core/src/structs/dollars.rs @@ -4,7 +4,9 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Cents; -#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +#[derive( + Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize, +)] pub struct Dollars(f64); impl From for Dollars { @@ -15,6 +17,12 @@ impl From for Dollars { impl From for Dollars { fn from(value: Cents) -> Self { - Self((*value as f64) / 100.0) + Self(f64::from(value) / 100.0) + } +} + +impl From for f64 { + fn from(value: Dollars) -> Self { + value.0 } } diff --git a/crates/brk_core/src/structs/feerate.rs b/crates/brk_core/src/structs/feerate.rs index 52ed2fad9..dc6c4c2d9 100644 --- a/crates/brk_core/src/structs/feerate.rs +++ b/crates/brk_core/src/structs/feerate.rs @@ -1,4 +1,2 @@ -use derive_deref::Deref; - -#[derive(Debug, Deref, Clone, Copy)] +#[derive(Debug, Clone, Copy)] pub struct Feerate(f32); diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index 4a4d11ae9..17757ecc2 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -1,19 +1,18 @@ use std::{ fmt::{self, Debug}, - ops::{Add, AddAssign, Rem, Sub}, + ops::{Add, AddAssign, Rem}, }; use bitcoincore_rpc::{Client, RpcApi}; -use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + #[derive( Debug, Clone, Copy, - Deref, - DerefMut, PartialEq, Eq, PartialOrd, @@ -49,11 +48,11 @@ impl Height { } pub fn decrement(&mut self) { - self.0 -= 1; + *self = self.decremented().unwrap(); } - pub fn decremented(self) -> Self { - Self(self.0.checked_sub(1).unwrap_or_default()) + pub fn decremented(self) -> Option { + self.checked_sub(1_u32) } pub fn is_zero(self) -> bool { @@ -63,7 +62,7 @@ impl Height { impl PartialEq for Height { fn eq(&self, other: &u64) -> bool { - **self == *other as u32 + self.0 == *other as u32 } } @@ -87,35 +86,25 @@ impl Add for Height { type Output = Height; fn add(self, rhs: usize) -> Self::Output { - Self::from(*self + rhs as u32) + Self::from(self.0 + rhs as u32) } } -impl Sub for Height { - type Output = Height; - fn sub(self, rhs: Height) -> Self::Output { - Self::from(*self - *rhs) +impl CheckedSub for Height { + fn checked_sub(self, rhs: Height) -> Option { + self.0.checked_sub(rhs.0).map(Height::from) } } -impl Sub for Height { - type Output = Height; - fn sub(self, rhs: i32) -> Self::Output { - Self::from(*self - rhs as u32) +impl CheckedSub for Height { + fn checked_sub(self, rhs: u32) -> Option { + self.0.checked_sub(rhs).map(Height::from) } } -impl Sub for Height { - type Output = Height; - fn sub(self, rhs: u32) -> Self::Output { - Self::from(*self - rhs) - } -} - -impl Sub for Height { - type Output = Height; - fn sub(self, rhs: usize) -> Self::Output { - Self::from(*self - rhs as u32) +impl CheckedSub for Height { + fn checked_sub(self, rhs: usize) -> Option { + self.0.checked_sub(rhs as u32).map(Height::from) } } @@ -141,7 +130,7 @@ impl Rem for Height { impl fmt::Display for Height { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", **self) + write!(f, "{}", self.0) } } @@ -168,6 +157,11 @@ impl From for usize { } } +impl From for u32 { + fn from(value: Height) -> Self { + value.0 + } +} impl From for u64 { fn from(value: Height) -> Self { value.0 as u64 @@ -189,7 +183,7 @@ impl From for Height { impl From for bitcoin::locktime::absolute::Height { fn from(value: Height) -> Self { - bitcoin::locktime::absolute::Height::from_consensus(*value).unwrap() + bitcoin::locktime::absolute::Height::from_consensus(value.0).unwrap() } } diff --git a/crates/brk_core/src/structs/locktime.rs b/crates/brk_core/src/structs/locktime.rs index 58a2ea9f2..3c436eed2 100644 --- a/crates/brk_core/src/structs/locktime.rs +++ b/crates/brk_core/src/structs/locktime.rs @@ -5,6 +5,7 @@ use super::{Height, Timestamp}; #[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)] #[repr(C)] +#[allow(warnings)] pub enum LockTime { Height(Height), Timestamp(Timestamp), diff --git a/crates/brk_core/src/structs/ohlc.rs b/crates/brk_core/src/structs/ohlc.rs index 3f78f1d04..0b55c89c1 100644 --- a/crates/brk_core/src/structs/ohlc.rs +++ b/crates/brk_core/src/structs/ohlc.rs @@ -2,9 +2,86 @@ use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::Cents; +use super::{Cents, Dollars}; -pub type OHLCCents = (Open, High, Low, Close); +#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +#[repr(C)] +pub struct OHLCCents { + pub open: Open, + pub high: High, + pub low: Low, + pub close: Close, +} + +impl From<(Open, High, Low, Close)> for OHLCCents { + fn from(value: (Open, High, Low, Close)) -> Self { + Self { + open: value.0, + high: value.1, + low: value.2, + close: value.3, + } + } +} + +impl From> for OHLCCents { + fn from(value: Close) -> Self { + Self { + open: Open::from(value), + high: High::from(value), + low: Low::from(value), + close: value, + } + } +} + +#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +#[repr(C)] +pub struct OHLCDollars { + pub open: Open, + pub high: High, + pub low: Low, + pub close: Close, +} + +impl From<(Open, High, Low, Close)> for OHLCDollars { + fn from(value: (Open, High, Low, Close)) -> Self { + Self { + open: value.0, + high: value.1, + low: value.2, + close: value.3, + } + } +} + +impl From> for OHLCDollars { + fn from(value: Close) -> Self { + Self { + open: Open::from(value), + high: High::from(value), + low: Low::from(value), + close: value, + } + } +} + +impl From for OHLCDollars { + fn from(value: OHLCCents) -> Self { + Self::from(&value) + } +} + +impl From<&OHLCCents> for OHLCDollars { + fn from(value: &OHLCCents) -> Self { + Self { + open: value.open.into(), + high: value.high.into(), + low: value.low.into(), + close: value.close.into(), + } + } +} #[derive( Debug, @@ -39,6 +116,12 @@ where } } +impl From> for Open { + fn from(value: Open) -> Self { + Self(Dollars::from(*value)) + } +} + #[derive( Debug, Default, @@ -72,6 +155,12 @@ where } } +impl From> for High { + fn from(value: High) -> Self { + Self(Dollars::from(*value)) + } +} + #[derive( Debug, Default, @@ -105,6 +194,12 @@ where } } +impl From> for Low { + fn from(value: Low) -> Self { + Self(Dollars::from(*value)) + } +} + #[derive( Debug, Default, @@ -128,3 +223,9 @@ impl From for Close { Self(value) } } + +impl From> for Close { + fn from(value: Close) -> Self { + Self(Dollars::from(*value)) + } +} diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs index 6c9c0d405..8d76dc973 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -4,7 +4,6 @@ use std::{ }; use bitcoin::Amount; -use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -18,8 +17,6 @@ use super::Height; Ord, Clone, Copy, - Deref, - DerefMut, Default, FromBytes, Immutable, @@ -40,7 +37,7 @@ impl Sats { impl Add for Sats { type Output = Sats; fn add(self, rhs: Sats) -> Self::Output { - Sats::from(*self + *rhs) + Sats::from(self.0 + rhs.0) } } @@ -53,7 +50,7 @@ impl AddAssign for Sats { impl Sub for Sats { type Output = Sats; fn sub(self, rhs: Sats) -> Self::Output { - Sats::from(*self - *rhs) + Sats::from(self.0 - rhs.0) } } @@ -66,27 +63,27 @@ impl SubAssign for Sats { impl Mul for Sats { type Output = Sats; fn mul(self, rhs: Sats) -> Self::Output { - Sats::from(*self * *rhs) + Sats::from(self.0 * rhs.0) } } impl Mul for Sats { type Output = Sats; fn mul(self, rhs: u64) -> Self::Output { - Sats::from(*self * rhs) + Sats::from(self.0 * rhs) } } impl Mul for Sats { type Output = Sats; fn mul(self, rhs: Height) -> Self::Output { - Sats::from(*self * *rhs as u64) + Sats::from(self.0 * u64::from(rhs)) } } impl Sum for Sats { fn sum>(iter: I) -> Self { - let sats: u64 = iter.map(|sats| *sats).sum(); + let sats: u64 = iter.map(|sats| sats.0).sum(); Sats::from(sats) } } @@ -107,3 +104,9 @@ impl From for Amount { Self::from_sat(value.0) } } + +impl From for f64 { + fn from(value: Sats) -> Self { + value.0 as f64 + } +} diff --git a/crates/brk_core/src/structs/txindex.rs b/crates/brk_core/src/structs/txindex.rs index bb080a3eb..3a956e84c 100644 --- a/crates/brk_core/src/structs/txindex.rs +++ b/crates/brk_core/src/structs/txindex.rs @@ -30,10 +30,6 @@ impl Txindex { pub fn incremented(self) -> Self { Self(*self + 1) } - - pub fn decremented(self) -> Self { - Self(*self - 1) - } } impl Add for Txindex { diff --git a/crates/brk_core/src/structs/txinindex.rs b/crates/brk_core/src/structs/txinindex.rs index 0089ea33e..40caf6062 100644 --- a/crates/brk_core/src/structs/txinindex.rs +++ b/crates/brk_core/src/structs/txinindex.rs @@ -29,10 +29,6 @@ impl Txinindex { pub fn incremented(self) -> Self { Self(*self + 1) } - - pub fn decremented(self) -> Self { - Self(*self - 1) - } } impl Add for Txinindex { diff --git a/crates/brk_core/src/structs/txoutindex.rs b/crates/brk_core/src/structs/txoutindex.rs index 72d4c0a37..e8560ac4c 100644 --- a/crates/brk_core/src/structs/txoutindex.rs +++ b/crates/brk_core/src/structs/txoutindex.rs @@ -32,10 +32,6 @@ impl Txoutindex { Self(*self + 1) } - pub fn decremented(self) -> Self { - Self(*self - 1) - } - pub fn is_coinbase(self) -> bool { self == Self::COINBASE } diff --git a/crates/brk_core/src/utils/checked_sub.rs b/crates/brk_core/src/utils/checked_sub.rs new file mode 100644 index 000000000..857d82881 --- /dev/null +++ b/crates/brk_core/src/utils/checked_sub.rs @@ -0,0 +1,3 @@ +pub trait CheckedSub: Sized { + fn checked_sub(self, rhs: Rhs) -> Option; +} diff --git a/crates/brk_core/src/utils/mod.rs b/crates/brk_core/src/utils/mod.rs index 189d0b677..04483c47f 100644 --- a/crates/brk_core/src/utils/mod.rs +++ b/crates/brk_core/src/utils/mod.rs @@ -1,7 +1,9 @@ +mod checked_sub; mod paths; mod pause; mod rlimit; +pub use checked_sub::*; pub use paths::*; pub use pause::*; pub use rlimit::*; diff --git a/crates/brk_core/src/utils/paths.rs b/crates/brk_core/src/utils/paths.rs index 41564a38d..647a54827 100644 --- a/crates/brk_core/src/utils/paths.rs +++ b/crates/brk_core/src/utils/paths.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + env, + path::{Path, PathBuf}, +}; pub fn path_dot_brk() -> PathBuf { let home = std::env::var("HOME").unwrap(); @@ -8,3 +11,26 @@ pub fn path_dot_brk() -> PathBuf { pub fn path_dot_brk_log() -> PathBuf { path_dot_brk().join("log") } + +pub fn default_brk() -> PathBuf { + path_dot_brk() +} + +pub fn default_bitcoin_path() -> PathBuf { + if env::consts::OS == "macos" { + default_mac_bitcoin_path() + } else { + default_linux_bitcoin_path() + } +} + +fn default_linux_bitcoin_path() -> PathBuf { + Path::new(&std::env::var("HOME").unwrap()).join(".bitcoin") +} + +fn default_mac_bitcoin_path() -> PathBuf { + Path::new(&std::env::var("HOME").unwrap()) + .join("Library") + .join("Application Support") + .join("Bitcoin") +} diff --git a/crates/brk_fetcher/examples/main.rs b/crates/brk_fetcher/examples/main.rs index 91817fb4d..e7ee777ea 100644 --- a/crates/brk_fetcher/examples/main.rs +++ b/crates/brk_fetcher/examples/main.rs @@ -10,7 +10,7 @@ fn main() -> color_eyre::Result<()> { dbg!(fetcher.get_date(Date::new(2025, 1, 1))?); dbg!(fetcher.get_height( - 880_000_u32.into(), + 881_000_u32.into(), 1740683986.into(), Some(1740683000.into()) )?); diff --git a/crates/brk_fetcher/src/fetchers/kibo.rs b/crates/brk_fetcher/src/fetchers/kibo.rs index 3182a65b4..5187f11e7 100644 --- a/crates/brk_fetcher/src/fetchers/kibo.rs +++ b/crates/brk_fetcher/src/fetchers/kibo.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, str::FromStr}; -use brk_core::{Date, Height, OHLCCents}; -use color_eyre::eyre::ContextCompat; +use brk_core::{CheckedSub, Date, Height, OHLCCents}; +use color_eyre::eyre::{ContextCompat, eyre}; use log::info; use serde_json::Value; @@ -14,56 +14,43 @@ pub struct Kibo { } const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api"; -const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api"; const RETRIES: usize = 10; impl Kibo { - fn get_base_url(try_index: usize) -> &'static str { - if try_index < RETRIES / 2 { - KIBO_OFFICIAL_URL - } else { - KIBO_OFFICIAL_BACKUP_URL - } - } - pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result { + let key = height.checked_sub(height % 10_000).unwrap_or_default(); + #[allow(clippy::map_entry)] - if !self.height_to_ohlc_vec.contains_key(&height) - || ((usize::from(height) + self.height_to_ohlc_vec.get(&height).unwrap().len()) - <= usize::from(height)) + if !self.height_to_ohlc_vec.contains_key(&key) + || ((key + self.height_to_ohlc_vec.get(&key).unwrap().len()) <= height) { self.height_to_ohlc_vec.insert( - height, - Self::fetch_height_prices(height).inspect_err(|e| { + key, + Self::fetch_height_prices(key).inspect_err(|e| { dbg!(e); })?, ); } - dbg!(&self.height_to_ohlc_vec.keys()); - self.height_to_ohlc_vec - .get(&height) + .get(&key) .unwrap() - .get(usize::from(height)) + .get(usize::from(height.checked_sub(key).unwrap())) .cloned() .ok_or(color_eyre::eyre::Error::msg("Couldn't find height in kibo")) } fn fetch_height_prices(height: Height) -> color_eyre::Result> { - info!("Fetching Kibo height prices..."); + info!("Fetching Kibo height {height} prices..."); retry( - |try_index| { - let base_url = Self::get_base_url(try_index); - - let url = format!("{base_url}/height-to-price?chunk={}", height); + |_| { + let url = format!("{KIBO_OFFICIAL_URL}/height-to-price?chunk={}", height); let body: Value = minreq::get(url).send()?.json()?; - let vec = body - .as_object() + body.as_object() .context("Expect to be an object")? .get("dataset") .context("Expect object to have dataset")? @@ -75,9 +62,7 @@ impl Kibo { .context("Expect to be an array")? .iter() .map(Self::value_to_ohlc) - .collect::, _>>()?; - - Ok(vec) + .collect::, _>>() }, 30, RETRIES, @@ -107,19 +92,21 @@ impl Kibo { .unwrap() .get(date) .cloned() - .ok_or(color_eyre::eyre::Error::msg("Couldn't find date in kibo")) + .ok_or({ + dbg!(date); + eyre!("Couldn't find date in kibo") + }) } fn fetch_date_prices(year: u16) -> color_eyre::Result> { - info!("Fetching Kibo date prices..."); + info!("Fetching Kibo date {year} prices..."); retry( - |try_index| { - let base_url = Self::get_base_url(try_index); - - let body: Value = minreq::get(format!("{base_url}/date-to-price?chunk={}", year)) - .send()? - .json()?; + |_| { + let body: Value = + minreq::get(format!("{KIBO_OFFICIAL_URL}/date-to-price?chunk={}", year)) + .send()? + .json()?; body.as_object() .context("Expect to be an object")? @@ -156,11 +143,11 @@ impl Kibo { ))) }; - Ok(( + Ok(OHLCCents::from(( Open::from(get_value("open")?), High::from(get_value("high")?), Low::from(get_value("low")?), Close::from(get_value("close")?), - )) + ))) } } diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index c6620c943..279113f27 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -106,12 +106,7 @@ impl Fetcher { let previous_ohlc = previous_ohlc.unwrap(); - let mut final_ohlc = ( - Open::from(previous_ohlc.3), - High::from(previous_ohlc.3), - Low::from(previous_ohlc.3), - previous_ohlc.3, - ); + let mut final_ohlc = OHLCCents::from(previous_ohlc.close); let start = previous_timestamp.unwrap_or(Timestamp::from(0)); let end = timestamp; @@ -119,15 +114,15 @@ impl Fetcher { // Otherwise it's a re-org if start < end { tree.range(start..=end).skip(1).for_each(|(_, ohlc)| { - if ohlc.1 > final_ohlc.1 { - final_ohlc.1 = ohlc.1 + if ohlc.high > final_ohlc.high { + final_ohlc.high = ohlc.high } - if ohlc.2 < final_ohlc.2 { - final_ohlc.2 = ohlc.2 + if ohlc.low < final_ohlc.low { + final_ohlc.low = ohlc.low } - final_ohlc.3 = ohlc.3; + final_ohlc.close = ohlc.close; }); } diff --git a/crates/brk_indexer/examples/main.rs b/crates/brk_indexer/examples/main.rs index 8935d6b2b..e7a86e6d7 100644 --- a/crates/brk_indexer/examples/main.rs +++ b/crates/brk_indexer/examples/main.rs @@ -1,5 +1,6 @@ use std::{path::Path, thread::sleep, time::Duration}; +use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_indexer::{Indexer, rpc::RpcApi}; use brk_parser::{ @@ -13,7 +14,8 @@ fn main() -> color_eyre::Result<()> { brk_logger::init(Some(Path::new(".log"))); - let bitcoin_dir = Path::new("../../../bitcoin"); + let bitcoin_dir = default_bitcoin_path(); + let rpc = Box::leak(Box::new(rpc::Client::new( "http://localhost:8332", rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index 80f30af14..d2d66d20c 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -1,7 +1,8 @@ use bitcoincore_rpc::Client; use brk_core::{ - Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex, - P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, + Addressindex, BlockHash, CheckedSub, Emptyindex, Height, Multisigindex, Opreturnindex, + P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, + Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, }; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; use color_eyre::eyre::ContextCompat; @@ -32,8 +33,10 @@ pub struct Indexes { impl Indexes { pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> { let height = self.height; - vecs.height_to_first_txindex.push_if_needed(height, self.txindex)?; - vecs.height_to_first_txinindex.push_if_needed(height, self.txinindex)?; + vecs.height_to_first_txindex + .push_if_needed(height, self.txindex)?; + vecs.height_to_first_txinindex + .push_if_needed(height, self.txinindex)?; vecs.height_to_first_txoutindex .push_if_needed(height, self.txoutindex)?; vecs.height_to_first_addressindex @@ -54,8 +57,10 @@ impl Indexes { .push_if_needed(height, self.p2pk65index)?; vecs.height_to_first_p2pkhindex .push_if_needed(height, self.p2pkhindex)?; - vecs.height_to_first_p2shindex.push_if_needed(height, self.p2shindex)?; - vecs.height_to_first_p2trindex.push_if_needed(height, self.p2trindex)?; + vecs.height_to_first_p2shindex + .push_if_needed(height, self.p2shindex)?; + vecs.height_to_first_p2trindex + .push_if_needed(height, self.p2trindex)?; vecs.height_to_first_p2wpkhindex .push_if_needed(height, self.p2wpkhindex)?; vecs.height_to_first_p2wshindex @@ -77,9 +82,11 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { // Height at which we wanna start: min last saved + 1 or 0 let starting_height = vecs.starting_height().min(stores.starting_height()); - let range = starting_height - .checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32) - .unwrap_or_default()..*starting_height; + let range = u32::from( + starting_height + .checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32) + .unwrap_or_default(), + )..u32::from(starting_height); // But we also need to check the chain and start earlier in case of a reorg let height = range // ..= because of last saved + 1 @@ -107,8 +114,14 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?, emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?, height, - multisigindex: *vecs.height_to_first_multisigindex.get(height)?.context("")?, - opreturnindex: *vecs.height_to_first_opreturnindex.get(height)?.context("")?, + multisigindex: *vecs + .height_to_first_multisigindex + .get(height)? + .context("")?, + opreturnindex: *vecs + .height_to_first_opreturnindex + .get(height)? + .context("")?, p2pk33index: *vecs.height_to_first_p2pk33index.get(height)?.context("")?, p2pk65index: *vecs.height_to_first_p2pk65index.get(height)?.context("")?, p2pkhindex: *vecs.height_to_first_p2pkhindex.get(height)?.context("")?, @@ -116,7 +129,10 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { p2trindex: *vecs.height_to_first_p2trindex.get(height)?.context("")?, p2wpkhindex: *vecs.height_to_first_p2wpkhindex.get(height)?.context("")?, p2wshindex: *vecs.height_to_first_p2wshindex.get(height)?.context("")?, - pushonlyindex: *vecs.height_to_first_pushonlyindex.get(height)?.context("")?, + pushonlyindex: *vecs + .height_to_first_pushonlyindex + .get(height)? + .context("")?, txindex: *vecs.height_to_first_txindex.get(height)?.context("")?, txinindex: *vecs.height_to_first_txinindex.get(height)?.context("")?, txoutindex: *vecs.height_to_first_txoutindex.get(height)?.context("")?, diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 25c032756..bc04a1769 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -83,7 +83,7 @@ impl Indexer { self.stores .as_mut() .unwrap() - .rollback_if_needed(self.vecs.as_ref().unwrap(), &starting_indexes)?; + .rollback_if_needed(self.vecs.as_mut().unwrap(), &starting_indexes)?; self.vecs .as_mut() .unwrap() diff --git a/crates/brk_indexer/src/stores/mod.rs b/crates/brk_indexer/src/stores/mod.rs index 653065117..a37336c0e 100644 --- a/crates/brk_indexer/src/stores/mod.rs +++ b/crates/brk_indexer/src/stores/mod.rs @@ -1,6 +1,9 @@ use std::{path::Path, thread}; -use brk_core::{AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix, Txindex}; +use brk_core::{ + AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix, + Txindex, +}; use brk_vec::{Value, Version}; use crate::Indexes; @@ -23,12 +26,14 @@ pub struct Stores { impl Stores { pub fn import(path: &Path) -> color_eyre::Result { thread::scope(|scope| { - let addresshash_to_addressindex = - scope.spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))); - let blockhash_prefix_to_height = - scope.spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))); - let txid_prefix_to_txindex = - scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1))); + let addresshash_to_addressindex = scope.spawn(|| { + Store::import(&path.join("addresshash_to_addressindex"), Version::from(1)) + }); + let blockhash_prefix_to_height = scope.spawn(|| { + Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1)) + }); + let txid_prefix_to_txindex = scope + .spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1))); Ok(Self { addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?, @@ -38,22 +43,29 @@ impl Stores { }) } - pub fn rollback_if_needed(&mut self, vecs: &Vecs, starting_indexes: &Indexes) -> color_eyre::Result<()> { + pub fn rollback_if_needed( + &mut self, + vecs: &mut Vecs, + starting_indexes: &Indexes, + ) -> color_eyre::Result<()> { vecs.height_to_blockhash - .iter_from(starting_indexes.height, |(_, blockhash)| { + .iter_from(starting_indexes.height, |(_, blockhash, ..)| { let blockhash_prefix = BlockHashPrefix::from(blockhash); self.blockhash_prefix_to_height.remove(blockhash_prefix); Ok(()) })?; vecs.txindex_to_txid - .iter_from(starting_indexes.txindex, |(_txindex, txid)| { + .iter_from(starting_indexes.txindex, |(_txindex, txid, ..)| { let txid_prefix = TxidPrefix::from(txid); self.txid_prefix_to_txindex.remove(txid_prefix); Ok(()) })?; - if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? { + if let Some(index) = vecs + .height_to_first_p2pk65index + .get(starting_indexes.height)? + { let mut index = index.into_inner(); while let Some(typedbytes) = vecs .p2pk65index_to_p2pk65addressbytes @@ -67,7 +79,10 @@ impl Stores { } } - if let Some(index) = vecs.height_to_first_p2pk33index.get(starting_indexes.height)? { + if let Some(index) = vecs + .height_to_first_p2pk33index + .get(starting_indexes.height)? + { let mut index = index.into_inner(); while let Some(typedbytes) = vecs .p2pk33index_to_p2pk33addressbytes @@ -81,9 +96,16 @@ impl Stores { } } - if let Some(index) = vecs.height_to_first_p2pkhindex.get(starting_indexes.height)? { + if let Some(index) = vecs + .height_to_first_p2pkhindex + .get(starting_indexes.height)? + { let mut index = index.into_inner(); - while let Some(typedbytes) = vecs.p2pkhindex_to_p2pkhaddressbytes.get(index)?.map(Value::into_inner) { + while let Some(typedbytes) = vecs + .p2pkhindex_to_p2pkhaddressbytes + .get(index)? + .map(Value::into_inner) + { let bytes = Addressbytes::from(typedbytes); let hash = AddressHash::from((&bytes, Addresstype::P2PKH)); self.addresshash_to_addressindex.remove(hash); @@ -91,9 +113,16 @@ impl Stores { } } - if let Some(index) = vecs.height_to_first_p2shindex.get(starting_indexes.height)? { + if let Some(index) = vecs + .height_to_first_p2shindex + .get(starting_indexes.height)? + { let mut index = index.into_inner(); - while let Some(typedbytes) = vecs.p2shindex_to_p2shaddressbytes.get(index)?.map(Value::into_inner) { + while let Some(typedbytes) = vecs + .p2shindex_to_p2shaddressbytes + .get(index)? + .map(Value::into_inner) + { let bytes = Addressbytes::from(typedbytes); let hash = AddressHash::from((&bytes, Addresstype::P2SH)); self.addresshash_to_addressindex.remove(hash); @@ -101,9 +130,16 @@ impl Stores { } } - if let Some(index) = vecs.height_to_first_p2trindex.get(starting_indexes.height)? { + if let Some(index) = vecs + .height_to_first_p2trindex + .get(starting_indexes.height)? + { let mut index = index.into_inner(); - while let Some(typedbytes) = vecs.p2trindex_to_p2traddressbytes.get(index)?.map(Value::into_inner) { + while let Some(typedbytes) = vecs + .p2trindex_to_p2traddressbytes + .get(index)? + .map(Value::into_inner) + { let bytes = Addressbytes::from(typedbytes); let hash = AddressHash::from((&bytes, Addresstype::P2TR)); self.addresshash_to_addressindex.remove(hash); @@ -111,7 +147,10 @@ impl Stores { } } - if let Some(index) = vecs.height_to_first_p2wpkhindex.get(starting_indexes.height)? { + if let Some(index) = vecs + .height_to_first_p2wpkhindex + .get(starting_indexes.height)? + { let mut index = index.into_inner(); while let Some(typedbytes) = vecs .p2wpkhindex_to_p2wpkhaddressbytes @@ -125,9 +164,16 @@ impl Stores { } } - if let Some(index) = vecs.height_to_first_p2wshindex.get(starting_indexes.height)? { + if let Some(index) = vecs + .height_to_first_p2wshindex + .get(starting_indexes.height)? + { let mut index = index.into_inner(); - while let Some(typedbytes) = vecs.p2wshindex_to_p2wshaddressbytes.get(index)?.map(Value::into_inner) { + while let Some(typedbytes) = vecs + .p2wshindex_to_p2wshaddressbytes + .get(index)? + .map(Value::into_inner) + { let bytes = Addressbytes::from(typedbytes); let hash = AddressHash::from((&bytes, Addresstype::P2WSH)); self.addresshash_to_addressindex.remove(hash); @@ -135,7 +181,7 @@ impl Stores { } } - self.commit(starting_indexes.height.decremented())?; + self.commit(starting_indexes.height.decremented().unwrap())?; Ok(()) } @@ -158,7 +204,8 @@ impl Stores { scope.spawn(|| self.addresshash_to_addressindex.commit(height)); let blockhash_prefix_to_height_commit_handle = scope.spawn(|| self.blockhash_prefix_to_height.commit(height)); - let txid_prefix_to_txindex_commit_handle = scope.spawn(|| self.txid_prefix_to_txindex.commit(height)); + let txid_prefix_to_txindex_commit_handle = + scope.spawn(|| self.txid_prefix_to_txindex.commit(height)); addresshash_to_addressindex_commit_handle.join().unwrap()?; blockhash_prefix_to_height_commit_handle.join().unwrap()?; diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index ab1aedcfd..c1806ed63 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -1,10 +1,11 @@ use std::{fs, io, path::Path}; use brk_core::{ - Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex, - Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex, - P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, - P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, + Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, + LockTime, Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, + P2PK65index, P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, + P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats, + Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, }; use brk_vec::{AnyStorableVec, Version}; use rayon::prelude::*; @@ -75,9 +76,18 @@ impl Vecs { &path.join("addressindex_to_addresstypeindex"), Version::from(1), )?, - addressindex_to_height: StorableVec::import(&path.join("addressindex_to_height"), Version::from(1))?, - height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?, - height_to_difficulty: StorableVec::import(&path.join("height_to_difficulty"), Version::from(1))?, + addressindex_to_height: StorableVec::import( + &path.join("addressindex_to_height"), + Version::from(1), + )?, + height_to_blockhash: StorableVec::import( + &path.join("height_to_blockhash"), + Version::from(1), + )?, + height_to_difficulty: StorableVec::import( + &path.join("height_to_difficulty"), + Version::from(1), + )?, height_to_first_addressindex: StorableVec::import( &path.join("height_to_first_addressindex"), Version::from(1), @@ -98,8 +108,14 @@ impl Vecs { &path.join("height_to_first_pushonlyindex"), Version::from(1), )?, - height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"), Version::from(1))?, - height_to_first_txinindex: StorableVec::import(&path.join("height_to_first_txinindex"), Version::from(1))?, + height_to_first_txindex: StorableVec::import( + &path.join("height_to_first_txindex"), + Version::from(1), + )?, + height_to_first_txinindex: StorableVec::import( + &path.join("height_to_first_txinindex"), + Version::from(1), + )?, height_to_first_txoutindex: StorableVec::import( &path.join("height_to_first_txoutindex"), Version::from(1), @@ -120,8 +136,14 @@ impl Vecs { &path.join("height_to_first_p2pkhindex"), Version::from(1), )?, - height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?, - height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?, + height_to_first_p2shindex: StorableVec::import( + &path.join("height_to_first_p2shindex"), + Version::from(1), + )?, + height_to_first_p2trindex: StorableVec::import( + &path.join("height_to_first_p2trindex"), + Version::from(1), + )?, height_to_first_p2wpkhindex: StorableVec::import( &path.join("height_to_first_p2wpkhindex"), Version::from(1), @@ -131,8 +153,14 @@ impl Vecs { Version::from(1), )?, height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?, - height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?, - height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?, + height_to_timestamp: StorableVec::import( + &path.join("height_to_timestamp"), + Version::from(1), + )?, + height_to_weight: StorableVec::import( + &path.join("height_to_weight"), + Version::from(1), + )?, p2pk33index_to_p2pk33addressbytes: StorableVec::import( &path.join("p2pk33index_to_p2pk33addressbytes"), Version::from(1), @@ -169,27 +197,48 @@ impl Vecs { &path.join("txindex_to_first_txoutindex"), Version::from(1), )?, - txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?, - txindex_to_locktime: StorableVec::import(&path.join("txindex_to_locktime"), Version::from(1))?, + txindex_to_height: StorableVec::import( + &path.join("txindex_to_height"), + Version::from(1), + )?, + txindex_to_locktime: StorableVec::import( + &path.join("txindex_to_locktime"), + Version::from(1), + )?, txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?, - txindex_to_base_size: StorableVec::import(&path.join("txindex_to_base_size"), Version::from(1))?, - txindex_to_total_size: StorableVec::import(&path.join("txindex_to_total_size"), Version::from(1))?, + txindex_to_base_size: StorableVec::import( + &path.join("txindex_to_base_size"), + Version::from(1), + )?, + txindex_to_total_size: StorableVec::import( + &path.join("txindex_to_total_size"), + Version::from(1), + )?, txindex_to_is_explicitly_rbf: StorableVec::import( &path.join("txindex_to_is_explicitly_rbf"), Version::from(1), )?, - txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?, - txinindex_to_txoutindex: StorableVec::import(&path.join("txinindex_to_txoutindex"), Version::from(1))?, + txindex_to_txversion: StorableVec::import( + &path.join("txindex_to_txversion"), + Version::from(1), + )?, + txinindex_to_txoutindex: StorableVec::import( + &path.join("txinindex_to_txoutindex"), + Version::from(1), + )?, txoutindex_to_addressindex: StorableVec::import( &path.join("txoutindex_to_addressindex"), Version::from(1), )?, - txoutindex_to_value: StorableVec::import(&path.join("txoutindex_to_value"), Version::from(1))?, + txoutindex_to_value: StorableVec::import( + &path.join("txoutindex_to_value"), + Version::from(1), + )?, }) } pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> { - let saved_height = starting_indexes.height.decremented(); + let saved_height = starting_indexes.height.decremented().unwrap_or_default(); // We don't want to override the starting indexes so we cut from n + 1 let height = starting_indexes.height.incremented(); @@ -218,7 +267,8 @@ impl Vecs { .truncate_if_needed(height, saved_height)?; self.height_to_first_pushonlyindex .truncate_if_needed(height, saved_height)?; - self.height_to_first_txindex.truncate_if_needed(height, saved_height)?; + self.height_to_first_txindex + .truncate_if_needed(height, saved_height)?; self.height_to_first_txinindex .truncate_if_needed(height, saved_height)?; self.height_to_first_txoutindex @@ -243,11 +293,16 @@ impl Vecs { .. } = starting_indexes; - self.height_to_blockhash.truncate_if_needed(height, saved_height)?; - self.height_to_difficulty.truncate_if_needed(height, saved_height)?; - self.height_to_size.truncate_if_needed(height, saved_height)?; - self.height_to_timestamp.truncate_if_needed(height, saved_height)?; - self.height_to_weight.truncate_if_needed(height, saved_height)?; + self.height_to_blockhash + .truncate_if_needed(height, saved_height)?; + self.height_to_difficulty + .truncate_if_needed(height, saved_height)?; + self.height_to_size + .truncate_if_needed(height, saved_height)?; + self.height_to_timestamp + .truncate_if_needed(height, saved_height)?; + self.height_to_weight + .truncate_if_needed(height, saved_height)?; self.addressindex_to_addresstype .truncate_if_needed(addressindex, saved_height)?; @@ -275,12 +330,18 @@ impl Vecs { .truncate_if_needed(txindex, saved_height)?; self.txindex_to_first_txoutindex .truncate_if_needed(txindex, saved_height)?; - self.txindex_to_height.truncate_if_needed(txindex, saved_height)?; - self.txindex_to_locktime.truncate_if_needed(txindex, saved_height)?; - self.txindex_to_txid.truncate_if_needed(txindex, saved_height)?; - self.txindex_to_txversion.truncate_if_needed(txindex, saved_height)?; - self.txindex_to_base_size.truncate_if_needed(txindex, saved_height)?; - self.txindex_to_total_size.truncate_if_needed(txindex, saved_height)?; + self.txindex_to_height + .truncate_if_needed(txindex, saved_height)?; + self.txindex_to_locktime + .truncate_if_needed(txindex, saved_height)?; + self.txindex_to_txid + .truncate_if_needed(txindex, saved_height)?; + self.txindex_to_txversion + .truncate_if_needed(txindex, saved_height)?; + self.txindex_to_base_size + .truncate_if_needed(txindex, saved_height)?; + self.txindex_to_total_size + .truncate_if_needed(txindex, saved_height)?; self.txindex_to_is_explicitly_rbf .truncate_if_needed(txindex, saved_height)?; @@ -289,7 +350,8 @@ impl Vecs { self.txoutindex_to_addressindex .truncate_if_needed(txoutindex, saved_height)?; - self.txoutindex_to_value.truncate_if_needed(txoutindex, saved_height)?; + self.txoutindex_to_value + .truncate_if_needed(txoutindex, saved_height)?; Ok(()) } @@ -351,13 +413,21 @@ impl Vecs { Addressbytes::P2PK33(bytes) => self .p2pk33index_to_p2pk33addressbytes .push_if_needed(index.into(), bytes), - Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index.into(), bytes), - Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index.into(), bytes), + Addressbytes::P2PKH(bytes) => self + .p2pkhindex_to_p2pkhaddressbytes + .push_if_needed(index.into(), bytes), + Addressbytes::P2SH(bytes) => self + .p2shindex_to_p2shaddressbytes + .push_if_needed(index.into(), bytes), Addressbytes::P2WPKH(bytes) => self .p2wpkhindex_to_p2wpkhaddressbytes .push_if_needed(index.into(), bytes), - Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index.into(), bytes), - Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index.into(), bytes), + Addressbytes::P2WSH(bytes) => self + .p2wshindex_to_p2wshaddressbytes + .push_if_needed(index.into(), bytes), + Addressbytes::P2TR(bytes) => self + .p2trindex_to_p2traddressbytes + .push_if_needed(index.into(), bytes), } } diff --git a/crates/brk_parser/examples/main.rs b/crates/brk_parser/examples/main.rs index 411a103e1..f0320a69c 100644 --- a/crates/brk_parser/examples/main.rs +++ b/crates/brk_parser/examples/main.rs @@ -1,13 +1,12 @@ -use std::path::Path; - use bitcoincore_rpc::{Auth, Client}; -use brk_core::Height; +use brk_core::{Height, default_bitcoin_path}; use brk_parser::Parser; fn main() { let i = std::time::Instant::now(); - let bitcoin_dir = Path::new("../../../bitcoin"); + let bitcoin_dir = default_bitcoin_path(); + let rpc = Box::leak(Box::new( Client::new( "http://localhost:8332", diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index 78d359f93..74c7457bb 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -23,8 +23,8 @@ use tree::VecIdToIndexToVec; pub struct Query<'a> { pub vecid_to_index_to_vec: VecIdToIndexToVec<'a>, - indexer: &'a Indexer, - computer: &'a Computer, + _indexer: &'a Indexer, + _computer: &'a Computer, } impl<'a> Query<'a> { @@ -45,8 +45,8 @@ impl<'a> Query<'a> { Self { vecid_to_index_to_vec: vecs, - indexer, - computer, + _indexer: indexer, + _computer: computer, } } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index d1da5ffa4..2b39c81aa 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -21,9 +21,9 @@ color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -oxc = { version = "0.56.0", features = ["codegen", "minifier"] } +oxc = { version = "0.56.5", features = ["codegen", "minifier"] } serde = { workspace = true } serde_json = { workspace = true } -tokio = { version = "1.43.0", features = ["full"] } +tokio = { version = "1.44.0", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full"] } zip = "2.2.3" diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 83b6ee16e..af4855a32 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -1,6 +1,7 @@ use std::{path::Path, thread::sleep, time::Duration}; use brk_computer::Computer; +use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::{ @@ -15,9 +16,10 @@ pub fn main() -> color_eyre::Result<()> { brk_logger::init(Some(Path::new(".log"))); - let process = false; + let process = true; + + let bitcoin_dir = default_bitcoin_path(); - let bitcoin_dir = Path::new("../../../bitcoin"); let rpc = Box::leak(Box::new(rpc::Client::new( "http://localhost:8332", rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index 890c1f7b2..45d841536 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true repository.workspace = true [dependencies] +brk_core = { workspace = true } brk_exit = { workspace = true } memmap2 = "0.9.5" rayon = { workspace = true } diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index 9db9dbe63..04533ad6b 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -16,6 +16,7 @@ use std::{ sync::OnceLock, }; +use brk_core::CheckedSub; use brk_exit::Exit; pub use memmap2; use rayon::prelude::*; @@ -258,6 +259,17 @@ where } #[inline] pub fn read_(&mut self, index: usize) -> Result> { + match self.index_to_pushed_index(index) { + Ok(index) => { + if let Some(index) = index { + return Ok(self.pushed.get(index)); + } + } + Err(Error::IndexTooHigh) => return Ok(None), + Err(Error::IndexTooLow) => {} + Err(error) => return Err(error), + } + let byte_index = Self::index_to_byte_index(index); if self.file_position != byte_index { self.file_position = self.seek(Self::index_to_byte_index(index))?; @@ -279,21 +291,20 @@ where self.read_(len - 1) } - pub fn iter(&self, f: F) -> Result<()> + pub fn iter(&mut self, f: F) -> Result<()> where - F: FnMut((I, &T)) -> Result<()>, + F: FnMut((I, &T, &mut Self)) -> Result<()>, { self.iter_from(I::default(), f) } - pub fn iter_from(&self, mut index: I, mut f: F) -> Result<()> + pub fn iter_from(&mut self, mut index: I, mut f: F) -> Result<()> where - F: FnMut((I, &T)) -> Result<()>, + F: FnMut((I, &T, &mut Self)) -> Result<()>, { let mut file = self.open_file()?; let disk_len = I::from(Self::read_disk_len_(&file)?); - let pushed_len = I::from(self.pushed_len()); Self::seek_( &mut file, @@ -303,18 +314,12 @@ where let mut buf = Self::create_buffer(); while index < disk_len { - f((index, Self::read_exact(&mut file, &mut buf)?))?; + f((index, Self::read_exact(&mut file, &mut buf)?, self))?; index = index + 1; } - let disk_len = Self::i_to_usize(disk_len)?; - let mut i = I::default(); - while i < pushed_len { - f(( - i + disk_len, - self.pushed.get(Self::i_to_usize(i)?).as_ref().unwrap(), - ))?; - i = i + 1; + if self.pushed_len() != 0 { + unreachable!(); } Ok(()) @@ -573,24 +578,26 @@ where std::any::type_name::() } - pub fn compute_transform( + pub fn compute_transform( &mut self, - max_from: I, - other: &mut StorableVec, - t: F, + max_from: A, + other: &mut StorableVec, + mut t: F, exit: &Exit, ) -> Result<()> where - A: StoredType, - F: Fn(&A, I) -> T, + A: StoredIndex, + B: StoredType, + F: FnMut((A, &B, &mut Self, &mut StorableVec)) -> (I, T), { self.validate_computed_version_or_reset_file( Version::from(0) + self.version + other.version, )?; - let index = max_from.min(I::from(self.len())); - other.iter_from(index, |(i, a)| { - self.push_and_flush_if_needed(i, t(a, i), exit) + let index = max_from.min(A::from(self.len())); + other.iter_from(index, |(a, b, other)| { + let (i, v) = t((a, b, self, other)); + self.push_and_flush_if_needed(i, v, exit) })?; Ok(self.safe_flush(exit)?) @@ -611,7 +618,14 @@ where )?; let index = max_from.min(self.read_last()?.cloned().unwrap_or_default()); - other.iter_from(index, |(v, i)| self.push_and_flush_if_needed(*i, v, exit))?; + other.iter_from(index, |(v, i, ..)| { + let i = *i; + if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) { + self.push_and_flush_if_needed(i, v, exit) + } else { + Ok(()) + } + })?; Ok(self.safe_flush(exit)?) } @@ -632,7 +646,7 @@ where )?; let index = max_from.min(T::from(self.len())); - first_indexes.iter_from(index, |(value, first_index)| { + first_indexes.iter_from(index, |(value, first_index, ..)| { let first_index = Self::i_to_usize(*first_index)?; let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?; (first_index..last_index) @@ -650,7 +664,7 @@ where exit: &Exit, ) -> Result<()> where - T: Copy + From + Sub + StoredIndex, + T: Copy + From + CheckedSub + StoredIndex, { self.validate_computed_version_or_reset_file( Version::from(0) + self.version + first_indexes.version, @@ -659,15 +673,19 @@ where let index = max_from.min(I::from(self.len())); let one = T::from(1); let mut prev_index: Option = None; - first_indexes.iter_from(index, |(i, v)| { + first_indexes.iter_from(index, |(i, v, ..)| { if let Some(prev_index) = prev_index { - self.push_and_flush_if_needed(prev_index, *v - one, exit)?; + self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?; } prev_index.replace(i); Ok(()) })?; if let Some(prev_index) = prev_index { - self.push_and_flush_if_needed(prev_index, T::from(final_len) - one, exit)?; + self.push_and_flush_if_needed( + prev_index, + T::from(final_len).checked_sub(one).unwrap(), + exit, + )?; } Ok(self.safe_flush(exit)?) @@ -690,7 +708,7 @@ where )?; let index = max_from.min(I::from(self.len())); - first_indexes.iter_from(index, |(i, first_index)| { + first_indexes.iter_from(index, |(i, first_index, ..)| { let last_index = last_indexes.read(i)?.unwrap(); let count = *last_index + 1_usize - *first_index; self.push_and_flush_if_needed(i, count.into(), exit) @@ -716,7 +734,7 @@ where )?; let index = max_from.min(I::from(self.len())); - self_to_other.iter_from(index, |(i, other)| { + self_to_other.iter_from(index, |(i, other, ..)| { self.push_and_flush_if_needed( i, T::from(other_to_self.read(*other)?.unwrap() == &i), @@ -745,7 +763,7 @@ where )?; let index = max_from.min(I::from(self.len())); - first_indexes.iter_from(index, |(index, first_index)| { + first_indexes.iter_from(index, |(index, first_index, ..)| { let last_index = last_indexes.read(index)?.unwrap(); let count = *last_index + 1_usize - *first_index; self.push_and_flush_if_needed(index, count.into(), exit)