diff --git a/Cargo.lock b/Cargo.lock index 176740a8e..52156bc39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "brk" -version = "0.0.9" +version = "0.0.10" dependencies = [ "brk_cli", "brk_computer", @@ -391,7 +391,7 @@ dependencies = [ [[package]] name = "brk_cli" -version = "0.0.9" +version = "0.0.10" dependencies = [ "brk_computer", "brk_core", @@ -412,7 +412,7 @@ dependencies = [ [[package]] name = "brk_computer" -version = "0.0.9" +version = "0.0.10" dependencies = [ "brk_core", "brk_exit", @@ -427,7 +427,7 @@ dependencies = [ [[package]] name = "brk_core" -version = "0.0.9" +version = "0.0.10" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -444,7 +444,7 @@ dependencies = [ [[package]] name = "brk_exit" -version = "0.0.9" +version = "0.0.10" dependencies = [ "brk_logger", "ctrlc", @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "brk_fetcher" -version = "0.0.9" +version = "0.0.10" dependencies = [ "brk_core", "brk_logger", @@ -466,7 +466,7 @@ dependencies = [ [[package]] name = "brk_indexer" -version = "0.0.9" +version = "0.0.10" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -485,7 +485,7 @@ dependencies = [ [[package]] name = "brk_logger" -version = "0.0.9" +version = "0.0.10" dependencies = [ "color-eyre", "env_logger", @@ -495,7 +495,7 @@ dependencies = [ [[package]] name = "brk_parser" -version = "0.0.9" +version = "0.0.10" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -510,7 +510,7 @@ dependencies = [ [[package]] name = "brk_query" -version = "0.0.9" +version = "0.0.10" dependencies = [ "brk_computer", "brk_indexer", @@ -526,7 +526,7 @@ dependencies = [ [[package]] name = "brk_server" -version = "0.0.9" +version = "0.0.10" dependencies = [ "axum", "brk_computer", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "brk_vec" -version = "0.0.9" +version = "0.0.10" dependencies = [ "memmap2", "rayon", @@ -764,9 +764,9 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "compact_str" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" dependencies = [ "castaway", "cfg-if", @@ -976,9 +976,9 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", "serde", @@ -1217,14 +1217,16 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1625,6 +1627,17 @@ dependencies = [ "crc", ] +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matchit" version = "0.8.4" @@ -1782,9 +1795,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "oxc" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45dac9dff4aa3da5b483ec7f7180b0af4a82882c3b35e67c8f9221e117bf0c93" +checksum = "3ceb3e662702022eb9714815de93b138d63e165c6773c1fcaf39e086ef9932ed" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1800,23 +1813,23 @@ dependencies = [ [[package]] name = "oxc-miette" -version = "1.0.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e03e63fd113c068b82d07c9c614b0b146c08a3ac0a4dface3ea1d1a9d14d549e" +checksum = "112f8458565c773a1f6555c14d6cd0255f56c85dd372932029e4baeb15308bbe" dependencies = [ "cfg-if", "owo-colors 4.2.0", "oxc-miette-derive", "textwrap", - "thiserror 1.0.69", + "thiserror", "unicode-width", ] [[package]] name = "oxc-miette-derive" -version = "1.0.2" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e21f680e8c5f1900297d394627d495351b9e37761f7bbf90116bd5eeb6e80967" +checksum = "4c0c893f53900e3fe01eca3d6d3b54085573c3e48fe25af9d57dd94ef600dcd3" dependencies = [ "proc-macro2", "quote", @@ -1825,9 +1838,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e49310ddfd3bc659d60b9f72bb0fbdb7b23f9bca5b4906056bf1d7d1a502d2" +checksum = "0fb8b78d9f70b5123bb01212458971da4cf0f796ad131602864df51d779290df" dependencies = [ "allocator-api2", "assert-unchecked", @@ -1839,9 +1852,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54af74d151e1a61d57ec8699f1e8b6729d3817fe763c2ecbacb945822998ea3e" +checksum = "d5cae7318a452457a303b688a1d12735e72e122114ac76bffa029ee34a5d0292" dependencies = [ "bitflags", "cow-utils", @@ -1856,9 +1869,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85874efff8c6b1f8b3adf8f3d8624e52ffab8a44da1e2e792de6a0303a9abb8" +checksum = "9ef30e196e021dcf6afd96417b9cd0fe223f0e517107aafdac9648022b280c1f" dependencies = [ "proc-macro2", "quote", @@ -1867,9 +1880,9 @@ dependencies = [ [[package]] name = "oxc_ast_visit" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bdf81b8db7952a841d15141e9efc40c8dd01720b9f1779b37f6d3ae5c9e7e4" +checksum = "1ac568f9ce9b9abf31b5f53d6c2f50b3bdb638c71539d177fb566a646cd32cf8" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1879,9 +1892,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc06be42ba66c3ab03fb82d973038ee8f5806cf8a7cd23beaa05f262cd63eee3" +checksum = "70c2580450c194b945007dbf2ed99ce167a9c61b0c6039d339c65735c35660fc" dependencies = [ "bitflags", "itertools", @@ -1894,9 +1907,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd09d5789bd90a760aa1e1a634fdd20e2b43981b590639264703e6760d33e5e6" +checksum = "cdb54f7e72e0e345e06a79a5c7807d365ab8b8b438742878a0ab8d3bdbba6a0e" dependencies = [ "bitflags", "cow-utils", @@ -1915,19 +1928,18 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92e1c4325cef51dda4296fd92302a6c3117325609efb81ac3f0996e7e44977b" +checksum = "1c5ff1571a7cbbaa30abd7dd624274b2fa67bf896d3121e8a0fff2ae50fa889a" dependencies = [ "assert-unchecked", - "ropey", ] [[package]] name = "oxc_diagnostics" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4866163037145687f7197fb70bad1fd0c109e9e2f70659f3eb7f038cd3168bd6" +checksum = "8c44fd92c6601ec6c5d02b210a8196027a0e49eebcc5ea7bcef2efe3c697ad64" dependencies = [ "cow-utils", "oxc-miette", @@ -1935,9 +1947,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fbfb5f543a10fb1264a5c24731ab700b5e6f7bedcc2c39792267039824216c" +checksum = "008ecf89e067b5ab8f5e1ac5890059f7c3cd5c9a8ae2db8ef4560dc93f53f01c" dependencies = [ "cow-utils", "num-bigint", @@ -1949,9 +1961,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84fade441037d1c4f5929f278f7ed74bc5b5928f35f5bc27ffb569512762622d" +checksum = "0e3ec988abe93acf6e99a5ddc3d08116977426fe64c78ed4d5d55852cdf13e14" [[package]] name = "oxc_index" @@ -1961,9 +1973,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfbf4a5f8bb51ec1b97100b4faaf50379f97d635bba0a39d95aa613c51fe630" +checksum = "be2f21973b9276723af3355830fb620922fa68963bb6617d0733555e98c16ba9" dependencies = [ "fixedbitset", "itertools", @@ -1978,9 +1990,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5c83bfaef4a1d5d8ffdf2b10bceadd57c8b15ccade74ae7a6672c6af31d095" +checksum = "4b6d676deac5a77f7fddadf01e8c1dff89bbe707c8e0f4ec1a734e8227458b5a" dependencies = [ "cow-utils", "oxc_allocator", @@ -2000,9 +2012,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b702c0462e5a67c845d7fafe236d8bb253ae0f4c8ff44a980b916a154862534" +checksum = "b70aae5a851f227bde01ffe9d6ad7b00cf1b836f62c9ea089da8372f4a30c78c" dependencies = [ "assert-unchecked", "bitflags", @@ -2023,9 +2035,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672d8601f80b9828342e0a38f6cc6735a04e2e4251c0c46476df55842df711a" +checksum = "7ed30ec210725f78e8f8c8d1cf597af11b40f18a62d4be5b97d3366ef0baee01" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -2039,9 +2051,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae29a491046d24b7cbac9ce0602e1c30c52f7065013e7212d716f283800a92ab" +checksum = "8a593327201c47df95293d5c2a85b13debd093b67c5a2ca4b5cbd6628a8e62dd" dependencies = [ "assert-unchecked", "itertools", @@ -2076,9 +2088,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c059e07f57c3299f54dfed3ba2f58dcc183ad68102d1186f8a4c5f546a2b9c5e" +checksum = "d1d4588fc61610c944557dc5b40a93ddbf497a98717f660f2258253ead7978a9" dependencies = [ "compact_str", "oxc-miette", @@ -2089,9 +2101,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2888043d4a47ee54903a229f3cfbab1126223c9b819505e900485993467a04d1" +checksum = "c8584c190328b5841abeeeba41342f4a187979eb75098fd506e522927c725211" dependencies = [ "assert-unchecked", "bitflags", @@ -2110,9 +2122,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.58.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b09630f1f467e91901e1451d651cb8fef0cab7b3facedecc022d6065842cd8a" +checksum = "53fa9743c952a78f5e66050a85f4dd60a549e305fb2aca565442186e7ac8db3c" dependencies = [ "compact_str", "itoa", @@ -2345,6 +2357,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "rand" version = "0.8.5" @@ -2462,16 +2480,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ropey" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93411e420bcd1a75ddd1dc3caf18c23155eda2c090631a85af21ba19e97093b5" -dependencies = [ - "smallvec", - "str_indices", -] - [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2775,12 +2783,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" -[[package]] -name = "str_indices" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" - [[package]] name = "strsim" version = "0.11.1" @@ -2851,7 +2853,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600" dependencies = [ "fastrand", - "getrandom 0.3.1", + "getrandom 0.3.2", "once_cell", "rustix", "windows-sys 0.59.0", @@ -2868,33 +2870,13 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - [[package]] name = "thiserror" version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.12", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "thiserror-impl", ] [[package]] @@ -2920,9 +2902,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.39" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8" +checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618" dependencies = [ "deranged", "itoa", @@ -2935,15 +2917,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c" +checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04" dependencies = [ "num-conv", "time-core", @@ -3205,9 +3187,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -3384,9 +3366,9 @@ dependencies = [ [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags", ] @@ -3397,6 +3379,15 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "zerocopy" version = "0.8.23" @@ -3439,9 +3430,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.2.3" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b280484c454e74e5fff658bbf7df8fdbe7a07c6b2de4a53def232c15ef138f3a" +checksum = "938cc23ac49778ac8340e366ddc422b2227ea176edb447e23fc0627608dddadd" dependencies = [ "aes", "arbitrary", @@ -3452,15 +3443,16 @@ dependencies = [ "deflate64", "displaydoc", "flate2", + "getrandom 0.3.2", "hmac", "indexmap 2.8.0", "lzma-rs", "memchr", "pbkdf2", - "rand", "sha1", - "thiserror 2.0.12", + "thiserror", "time", + "xz2", "zeroize", "zopfli", "zstd", diff --git a/Cargo.toml b/Cargo.toml index 4d13b97c9..161d93782 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" package.edition = "2024" -package.version = "0.0.9" +package.version = "0.0.10" package.repository = "https://github.com/bitcoinresearchkit/brk" [workspace.dependencies] diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index 0191e2f8e..b4b7688f2 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -5,10 +5,7 @@ use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_parser::{ - Parser, - rpc::{self}, -}; +use brk_parser::{Parser, rpc}; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs index d95b7118e..f9070b64a 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -3,13 +3,13 @@ use std::{ cmp::Ordering, fmt::Debug, io, - ops::{Add, Deref, DerefMut, Sub}, + ops::{Add, Sub}, path::{Path, PathBuf}, }; use brk_core::CheckedSub; use brk_exit::Exit; -use brk_vec::{Compressed, Error, Result, StoredIndex, StoredType, Version}; +use brk_vec::{AnyStorableVec, Compressed, Error, Result, StoredIndex, StoredType, Version}; const FLUSH_EVERY: usize = 10_000; @@ -24,7 +24,11 @@ where I: StoredIndex, T: StoredType, { - pub fn import(path: &Path, version: Version, compressed: Compressed) -> brk_vec::Result { + pub fn forced_import( + path: &Path, + version: Version, + compressed: Compressed, + ) -> brk_vec::Result { let vec = brk_vec::StorableVec::forced_import(path, version, compressed)?; Ok(Self { @@ -33,24 +37,19 @@ where }) } - #[inline] - pub fn i_to_usize(index: I) -> Result { - index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize) - } - fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> { if exit.triggered() { return Ok(()); } exit.block(); - self.truncate_if_needed(index)?; + self.vec.truncate_if_needed(index)?; exit.release(); Ok(()) } #[inline] - fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> { - match self.len().cmp(&Self::i_to_usize(index)?) { + pub fn forced_push_at(&mut self, index: I, value: T, exit: &Exit) -> Result<()> { + match self.len().cmp(&index.to_usize()?) { Ordering::Less => { return Err(Error::IndexTooHigh); } @@ -58,11 +57,11 @@ where if ord == Ordering::Greater { self.safe_truncate_if_needed(index, exit)?; } - self.push(value); + self.vec.push(value); } } - if self.pushed_len() >= FLUSH_EVERY { + if self.vec.pushed_len() >= FLUSH_EVERY { Ok(self.safe_flush(exit)?) } else { Ok(()) @@ -74,20 +73,48 @@ where return Ok(()); } exit.block(); - self.flush()?; + self.vec.flush()?; exit.release(); Ok(()) } + fn version(&self) -> Version { + self.vec.version() + } + + fn len(&self) -> usize { + self.vec.len() + } + + pub fn vec(&self) -> &brk_vec::StorableVec { + &self.vec + } + + pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec { + &mut self.vec + } + + pub fn any_vec(&self) -> &dyn AnyStorableVec { + &self.vec + } + + pub fn get(&mut self, index: I) -> Result> { + self.vec.get(index) + } + + pub fn collect_range(&self, from: Option, to: Option) -> Result> { + self.vec.collect_range(from, to) + } + #[inline] fn path_computed_version(&self) -> PathBuf { - self.path().join("computed_version") + self.vec.path().join("computed_version") } fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { let path = self.path_computed_version(); if version.validate(path.as_ref()).is_err() { - self.reset()?; + self.vec.reset()?; } version.write(path.as_ref())?; Ok(()) @@ -112,7 +139,7 @@ where let index = max_from.min(A::from(self.len())); other.iter_from_cloned(index, |(a, b, other)| { let (i, v) = t((a, b, self, other)); - self.push_and_flush_if_needed(i, v, exit) + self.forced_push_at(i, v, exit) })?; Ok(self.safe_flush(exit)?) @@ -132,11 +159,11 @@ where Version::from(0) + self.version() + other.version(), )?; - let index = max_from.min(self.read_last()?.cloned().unwrap_or_default()); + let index = max_from.min(self.vec.get_last()?.cloned().unwrap_or_default()); other.iter_from(index, |(v, i, ..)| { let i = *i; - if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) { - self.push_and_flush_if_needed(i, v, exit) + if self.get(i).unwrap().is_none_or(|old_v| *old_v > v) { + self.forced_push_at(i, v, exit) } else { Ok(()) } @@ -162,10 +189,10 @@ where let index = max_from.min(T::from(self.len())); first_indexes.iter_from(index, |(value, first_index, ..)| { - let first_index = Self::i_to_usize(*first_index)?; - let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?; + let first_index = (first_index).to_usize()?; + let last_index = (last_indexes.get(value)?.unwrap()).to_usize()?; (first_index..last_index) - .try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit)) + .try_for_each(|index| self.forced_push_at(I::from(index), value, exit)) })?; Ok(self.safe_flush(exit)?) @@ -190,13 +217,13 @@ where let mut prev_index: Option = None; first_indexes.iter_from(index, |(i, v, ..)| { if let Some(prev_index) = prev_index.take() { - self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?; + self.forced_push_at(prev_index, v.checked_sub(one).unwrap(), exit)?; } prev_index.replace(i); Ok(()) })?; if let Some(prev_index) = prev_index { - self.push_and_flush_if_needed( + self.forced_push_at( prev_index, T::from(final_len).checked_sub(one).unwrap(), exit, @@ -215,7 +242,7 @@ where ) -> Result<()> where T: From, - T2: StoredType + Copy + Add + Sub + TryInto, + T2: StoredType + Copy + Add + CheckedSub + TryInto + Default, >::Error: error::Error + 'static, { self.validate_computed_version_or_reset_file( @@ -224,9 +251,11 @@ where let index = max_from.min(I::from(self.len())); first_indexes.iter_from(index, |(i, first_index, ..)| { - let last_index = last_indexes.read(i)?.unwrap(); - let count = *last_index + 1_usize - *first_index; - self.push_and_flush_if_needed(i, count.into(), exit) + let last_index = last_indexes.get(i)?.unwrap(); + let count = (*last_index + 1_usize) + .checked_sub(*first_index) + .unwrap_or_default(); + self.forced_push_at(i, count.into(), exit) })?; Ok(self.safe_flush(exit)?) @@ -250,11 +279,7 @@ where let index = max_from.min(I::from(self.len())); self_to_other.iter_from(index, |(i, other, ..)| { - self.push_and_flush_if_needed( - i, - T::from(other_to_self.read(*other)?.unwrap() == &i), - exit, - ) + self.forced_push_at(i, T::from(other_to_self.get(*other)?.unwrap() == &i), exit) })?; Ok(self.safe_flush(exit)?) @@ -278,26 +303,15 @@ where let index = max_from.min(I::from(self.len())); first_indexes.iter_from(index, |(index, first_index, ..)| { - let last_index = last_indexes.read(index)?.unwrap(); + let last_index = last_indexes.get(index)?.unwrap(); let count = *last_index + 1_usize - *first_index; - self.push_and_flush_if_needed(index, count.into(), exit) + self.forced_push_at(index, count.into(), exit) })?; Ok(self.safe_flush(exit)?) } } -impl Deref for StorableVec { - type Target = brk_vec::StorableVec; - fn deref(&self) -> &Self::Target { - &self.vec - } -} -impl DerefMut for StorableVec { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.vec - } -} impl Clone for StorableVec where I: StoredIndex, diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs new file mode 100644 index 000000000..64448d708 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -0,0 +1,99 @@ +use std::{fs, path::Path}; + +use brk_core::{CheckedSub, Height, Timestamp}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyStorableVec, Compressed, Version}; + +use super::{ + Indexes, StorableVec, indexes, + stats::{StorableVecGeneatorByIndex, StorableVecGeneatorOptions}, +}; + +#[derive(Clone)] +pub struct Vecs { + pub height_to_block_interval: StorableVec, + pub indexes_to_block_interval_stats: StorableVecGeneatorByIndex, + pub dateindex_to_block_count: StorableVec, + pub dateindex_to_total_block_count: StorableVec, +} + +impl Vecs { + pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + height_to_block_interval: StorableVec::forced_import( + &path.join("height_to_block_interval"), + Version::from(1), + compressed, + )?, + indexes_to_block_interval_stats: StorableVecGeneatorByIndex::forced_import( + &path.join("block_interval"), + compressed, + StorableVecGeneatorOptions::default() + .add_percentiles() + .add_minmax() + .add_average(), + )?, + dateindex_to_block_count: StorableVec::forced_import( + &path.join("dateindex_to_block_count"), + Version::from(1), + compressed, + )?, + dateindex_to_total_block_count: StorableVec::forced_import( + &path.join("dateindex_to_total_block_count"), + Version::from(1), + compressed, + )?, + }) + } + + pub fn compute( + &mut self, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> { + let indexer_vecs = indexer.mut_vecs(); + + self.height_to_block_interval.compute_transform( + starting_indexes.height, + indexer_vecs.height_to_timestamp.mut_vec(), + |(height, timestamp, height_to_timestamp, ..)| { + let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| { + let prev_timestamp = *height_to_timestamp.get(prev_h).unwrap().unwrap(); + if prev_timestamp >= timestamp { + Timestamp::ZERO + } else { + timestamp.checked_sub(prev_timestamp).unwrap() + } + }); + (height, interval) + }, + exit, + )?; + + self.indexes_to_block_interval_stats.compute( + &mut self.height_to_block_interval, + indexer, + indexes, + starting_indexes, + ); + + Ok(()) + } + + pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + [ + vec![ + self.height_to_block_interval.any_vec(), + self.dateindex_to_block_count.any_vec(), + self.dateindex_to_total_block_count.any_vec(), + ], + self.indexes_to_block_interval_stats.as_any_vecs(), + ] + .concat() + } +} diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 964422640..963a4a0d9 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -3,7 +3,7 @@ use std::{fs, ops::Deref, path::Path}; use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, Compressed, Value, Version}; +use brk_vec::{AnyStorableVec, Compressed, Version}; use super::StorableVec; @@ -25,62 +25,62 @@ pub struct Vecs { } impl Vecs { - pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { - dateindex_to_date: StorableVec::import( + dateindex_to_date: StorableVec::forced_import( &path.join("dateindex_to_date"), Version::from(1), compressed, )?, - dateindex_to_dateindex: StorableVec::import( + dateindex_to_dateindex: StorableVec::forced_import( &path.join("dateindex_to_dateindex"), Version::from(1), compressed, )?, - dateindex_to_first_height: StorableVec::import( + dateindex_to_first_height: StorableVec::forced_import( &path.join("dateindex_to_first_height"), Version::from(1), compressed, )?, - dateindex_to_last_height: StorableVec::import( + dateindex_to_last_height: StorableVec::forced_import( &path.join("dateindex_to_last_height"), Version::from(1), compressed, )?, - height_to_real_date: StorableVec::import( + height_to_real_date: StorableVec::forced_import( &path.join("height_to_real_date"), Version::from(1), compressed, )?, - height_to_fixed_date: StorableVec::import( + height_to_fixed_date: StorableVec::forced_import( &path.join("height_to_fixed_date"), Version::from(1), compressed, )?, - height_to_dateindex: StorableVec::import( + height_to_dateindex: StorableVec::forced_import( &path.join("height_to_dateindex"), Version::from(1), compressed, )?, - height_to_height: StorableVec::import( + height_to_height: StorableVec::forced_import( &path.join("height_to_height"), Version::from(1), compressed, )?, - height_to_last_txindex: StorableVec::import( + height_to_last_txindex: StorableVec::forced_import( &path.join("height_to_last_txindex"), Version::from(1), compressed, )?, - txindex_to_last_txinindex: StorableVec::import( + txindex_to_last_txinindex: StorableVec::forced_import( &path.join("txindex_to_last_txinindex"), Version::from(1), compressed, )?, - txindex_to_last_txoutindex: StorableVec::import( + txindex_to_last_txoutindex: StorableVec::forced_import( &path.join("txindex_to_last_txoutindex"), Version::from(1), compressed, @@ -103,25 +103,25 @@ impl Vecs { self.height_to_height.compute_transform( starting_indexes.height, - &mut indexer_vecs.height_to_timestamp, + indexer_vecs.height_to_timestamp.mut_vec(), |(h, ..)| (h, h), exit, )?; self.height_to_real_date.compute_transform( starting_indexes.height, - &mut indexer_vecs.height_to_timestamp, + indexer_vecs.height_to_timestamp.mut_vec(), |(h, t, ..)| (h, Date::from(t)), exit, )?; self.height_to_fixed_date.compute_transform( starting_indexes.height, - &mut self.height_to_real_date, + self.height_to_real_date.mut_vec(), |(h, d, s, ..)| { let d = h .decremented() - .and_then(|h| s.read(h).ok()) + .and_then(|h| s.get(h).ok()) .flatten() .map_or(d, |prev_d| { let prev_d = *prev_d; @@ -134,7 +134,7 @@ impl Vecs { self.height_to_dateindex.compute_transform( starting_indexes.height, - &mut self.height_to_fixed_date, + self.height_to_fixed_date.mut_vec(), |(h, d, ..)| (h, Dateindex::try_from(d).unwrap()), exit, )?; @@ -142,34 +142,34 @@ impl Vecs { let starting_dateindex = self .height_to_dateindex .get(starting_indexes.height.decremented().unwrap_or_default())? - .map(Value::into_inner) + .copied() .unwrap_or_default(); self.dateindex_to_first_height .compute_inverse_more_to_less( starting_indexes.height, - &mut self.height_to_dateindex, + self.height_to_dateindex.mut_vec(), exit, )?; self.dateindex_to_last_height .compute_last_index_from_first( starting_dateindex, - &mut self.dateindex_to_first_height, + self.dateindex_to_first_height.mut_vec(), height_count, exit, )?; self.dateindex_to_dateindex.compute_transform( starting_dateindex, - &mut self.dateindex_to_first_height, + self.dateindex_to_first_height.mut_vec(), |(di, ..)| (di, di), exit, )?; self.dateindex_to_date.compute_transform( starting_dateindex, - &mut self.dateindex_to_dateindex, + self.dateindex_to_dateindex.mut_vec(), |(di, ..)| (di, Date::from(di)), exit, )?; @@ -177,7 +177,7 @@ impl Vecs { self.txindex_to_last_txinindex .compute_last_index_from_first( starting_indexes.txindex, - &mut indexer_vecs.txindex_to_first_txinindex, + indexer_vecs.txindex_to_first_txinindex.mut_vec(), txinindexes_count, exit, )?; @@ -185,14 +185,14 @@ impl Vecs { self.txindex_to_last_txoutindex .compute_last_index_from_first( starting_indexes.txindex, - &mut indexer_vecs.txindex_to_first_txoutindex, + indexer_vecs.txindex_to_first_txoutindex.mut_vec(), txoutindexes_count, exit, )?; self.height_to_last_txindex.compute_last_index_from_first( starting_indexes.height, - &mut indexer_vecs.height_to_first_txindex, + indexer_vecs.height_to_first_txindex.mut_vec(), txindexes_count, exit, )?; @@ -202,17 +202,17 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { vec![ - &*self.dateindex_to_date, - &*self.dateindex_to_dateindex, - &*self.dateindex_to_first_height, - &*self.dateindex_to_last_height, - &*self.height_to_dateindex, - &*self.height_to_fixed_date, - &*self.height_to_height, - &*self.height_to_last_txindex, - &*self.height_to_real_date, - &*self.txindex_to_last_txinindex, - &*self.txindex_to_last_txoutindex, + self.dateindex_to_date.any_vec(), + self.dateindex_to_dateindex.any_vec(), + self.dateindex_to_first_height.any_vec(), + self.dateindex_to_last_height.any_vec(), + self.height_to_dateindex.any_vec(), + self.height_to_fixed_date.any_vec(), + self.height_to_height.any_vec(), + self.height_to_last_txindex.any_vec(), + self.height_to_real_date.any_vec(), + self.txindex_to_last_txinindex.any_vec(), + self.txindex_to_last_txoutindex.any_vec(), ] } } diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index 9dc7bc4f4..47a101abb 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -6,7 +6,7 @@ use brk_core::{ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, Compressed, Value, Version}; +use brk_vec::{AnyStorableVec, Compressed, Version}; use super::{Indexes, StorableVec, indexes}; @@ -37,116 +37,116 @@ pub struct Vecs { } impl Vecs { - pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { - dateindex_to_ohlc_in_cents: StorableVec::import( + dateindex_to_ohlc_in_cents: StorableVec::forced_import( &path.join("dateindex_to_ohlc_in_cents"), Version::from(1), compressed, )?, - dateindex_to_ohlc: StorableVec::import( + dateindex_to_ohlc: StorableVec::forced_import( &path.join("dateindex_to_ohlc"), Version::from(1), compressed, )?, - dateindex_to_close_in_cents: StorableVec::import( + dateindex_to_close_in_cents: StorableVec::forced_import( &path.join("dateindex_to_close_in_cents"), Version::from(1), compressed, )?, - dateindex_to_close: StorableVec::import( + dateindex_to_close: StorableVec::forced_import( &path.join("dateindex_to_close"), Version::from(1), compressed, )?, - dateindex_to_high_in_cents: StorableVec::import( + dateindex_to_high_in_cents: StorableVec::forced_import( &path.join("dateindex_to_high_in_cents"), Version::from(1), compressed, )?, - dateindex_to_high: StorableVec::import( + dateindex_to_high: StorableVec::forced_import( &path.join("dateindex_to_high"), Version::from(1), compressed, )?, - dateindex_to_low_in_cents: StorableVec::import( + dateindex_to_low_in_cents: StorableVec::forced_import( &path.join("dateindex_to_low_in_cents"), Version::from(1), compressed, )?, - dateindex_to_low: StorableVec::import( + dateindex_to_low: StorableVec::forced_import( &path.join("dateindex_to_low"), Version::from(1), compressed, )?, - dateindex_to_open_in_cents: StorableVec::import( + dateindex_to_open_in_cents: StorableVec::forced_import( &path.join("dateindex_to_open_in_cents"), Version::from(1), compressed, )?, - dateindex_to_open: StorableVec::import( + dateindex_to_open: StorableVec::forced_import( &path.join("dateindex_to_open"), Version::from(1), compressed, )?, - dateindex_to_sats_per_dollar: StorableVec::import( + dateindex_to_sats_per_dollar: StorableVec::forced_import( &path.join("dateindex_to_sats_per_dollar"), Version::from(1), compressed, )?, - height_to_ohlc_in_cents: StorableVec::import( + height_to_ohlc_in_cents: StorableVec::forced_import( &path.join("height_to_ohlc_in_cents"), Version::from(1), compressed, )?, - height_to_ohlc: StorableVec::import( + height_to_ohlc: StorableVec::forced_import( &path.join("height_to_ohlc"), Version::from(1), compressed, )?, - height_to_close_in_cents: StorableVec::import( + height_to_close_in_cents: StorableVec::forced_import( &path.join("height_to_close_in_cents"), Version::from(1), compressed, )?, - height_to_close: StorableVec::import( + height_to_close: StorableVec::forced_import( &path.join("height_to_close"), Version::from(1), compressed, )?, - height_to_high_in_cents: StorableVec::import( + height_to_high_in_cents: StorableVec::forced_import( &path.join("height_to_high_in_cents"), Version::from(1), compressed, )?, - height_to_high: StorableVec::import( + height_to_high: StorableVec::forced_import( &path.join("height_to_high"), Version::from(1), compressed, )?, - height_to_low_in_cents: StorableVec::import( + height_to_low_in_cents: StorableVec::forced_import( &path.join("height_to_low_in_cents"), Version::from(1), compressed, )?, - height_to_low: StorableVec::import( + height_to_low: StorableVec::forced_import( &path.join("height_to_low"), Version::from(1), compressed, )?, - height_to_open_in_cents: StorableVec::import( + height_to_open_in_cents: StorableVec::forced_import( &path.join("height_to_open_in_cents"), Version::from(1), compressed, )?, - height_to_open: StorableVec::import( + height_to_open: StorableVec::forced_import( &path.join("height_to_open"), Version::from(1), compressed, )?, - height_to_sats_per_dollar: StorableVec::import( + height_to_sats_per_dollar: StorableVec::forced_import( &path.join("height_to_sats_per_dollar"), Version::from(1), compressed, @@ -166,19 +166,14 @@ impl Vecs { self.height_to_ohlc_in_cents.compute_transform( starting_indexes.height, - &mut indexer_vecs.height_to_timestamp, + indexer_vecs.height_to_timestamp.mut_vec(), |(h, t, _, height_to_timestamp)| { let ohlc = fetcher .get_height( h, t, - h.decremented().map(|prev_h| { - height_to_timestamp - .get(prev_h) - .unwrap() - .map(Value::into_inner) - .unwrap() - }), + h.decremented() + .map(|prev_h| *height_to_timestamp.get(prev_h).unwrap().unwrap()), ) .unwrap(); (h, ohlc) @@ -188,77 +183,77 @@ impl Vecs { self.height_to_open_in_cents.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc_in_cents, + self.height_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, )?; self.height_to_high_in_cents.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc_in_cents, + self.height_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, )?; self.height_to_low_in_cents.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc_in_cents, + self.height_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, )?; self.height_to_close_in_cents.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc_in_cents, + self.height_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, )?; self.height_to_ohlc.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc_in_cents, + self.height_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), exit, )?; self.height_to_open.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc, + self.height_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, )?; self.height_to_high.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc, + self.height_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, )?; self.height_to_low.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc, + self.height_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, )?; self.height_to_close.compute_transform( starting_indexes.height, - &mut self.height_to_ohlc, + self.height_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, )?; self.height_to_sats_per_dollar.compute_transform( starting_indexes.height, - &mut self.height_to_close, + self.height_to_close.mut_vec(), |(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)), exit, )?; self.dateindex_to_ohlc_in_cents.compute_transform( starting_indexes.dateindex, - &mut indexes.dateindex_to_date, + indexes.dateindex_to_date.mut_vec(), |(di, d, ..)| { let ohlc = fetcher.get_date(d).unwrap(); (di, ohlc) @@ -268,70 +263,70 @@ impl Vecs { self.dateindex_to_open_in_cents.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc_in_cents, + self.dateindex_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, )?; self.dateindex_to_high_in_cents.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc_in_cents, + self.dateindex_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, )?; self.dateindex_to_low_in_cents.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc_in_cents, + self.dateindex_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, )?; self.dateindex_to_close_in_cents.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc_in_cents, + self.dateindex_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, )?; self.dateindex_to_ohlc.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc_in_cents, + self.dateindex_to_ohlc_in_cents.mut_vec(), |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), exit, )?; self.dateindex_to_open.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc, + self.dateindex_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, )?; self.dateindex_to_high.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc, + self.dateindex_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, )?; self.dateindex_to_low.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc, + self.dateindex_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, )?; self.dateindex_to_close.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_ohlc, + self.dateindex_to_ohlc.mut_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, )?; self.dateindex_to_sats_per_dollar.compute_transform( starting_indexes.dateindex, - &mut self.dateindex_to_close, + self.dateindex_to_close.mut_vec(), |(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)), exit, )?; @@ -341,28 +336,28 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { vec![ - &*self.dateindex_to_close, - &*self.dateindex_to_close_in_cents, - &*self.dateindex_to_high, - &*self.dateindex_to_high_in_cents, - &*self.dateindex_to_low, - &*self.dateindex_to_low_in_cents, - &*self.dateindex_to_ohlc, - &*self.dateindex_to_ohlc_in_cents, - &*self.dateindex_to_open, - &*self.dateindex_to_open_in_cents, - &*self.dateindex_to_sats_per_dollar, - &*self.height_to_close, - &*self.height_to_close_in_cents, - &*self.height_to_high, - &*self.height_to_high_in_cents, - &*self.height_to_low, - &*self.height_to_low_in_cents, - &*self.height_to_ohlc, - &*self.height_to_ohlc_in_cents, - &*self.height_to_open, - &*self.height_to_open_in_cents, - &*self.height_to_sats_per_dollar, + self.dateindex_to_close.any_vec(), + self.dateindex_to_close_in_cents.any_vec(), + self.dateindex_to_high.any_vec(), + self.dateindex_to_high_in_cents.any_vec(), + self.dateindex_to_low.any_vec(), + self.dateindex_to_low_in_cents.any_vec(), + self.dateindex_to_ohlc.any_vec(), + self.dateindex_to_ohlc_in_cents.any_vec(), + self.dateindex_to_open.any_vec(), + self.dateindex_to_open_in_cents.any_vec(), + self.dateindex_to_sats_per_dollar.any_vec(), + self.height_to_close.any_vec(), + self.height_to_close_in_cents.any_vec(), + self.height_to_high.any_vec(), + self.height_to_high_in_cents.any_vec(), + self.height_to_low.any_vec(), + self.height_to_low_in_cents.any_vec(), + self.height_to_ohlc.any_vec(), + self.height_to_ohlc_in_cents.any_vec(), + self.height_to_open.any_vec(), + self.height_to_open_in_cents.any_vec(), + self.height_to_sats_per_dollar.any_vec(), ] } } diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index 393f33bdc..2b2418792 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -6,8 +6,10 @@ use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed}; mod base; +mod blocks; mod indexes; mod marketprice; +mod stats; mod transactions; use base::*; @@ -15,6 +17,7 @@ use indexes::*; #[derive(Clone)] pub struct Vecs { + pub blocks: blocks::Vecs, pub indexes: indexes::Vecs, pub transactions: transactions::Vecs, pub marketprice: Option, @@ -25,9 +28,10 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - indexes: indexes::Vecs::import(path, compressed)?, - transactions: transactions::Vecs::import(path, compressed)?, - marketprice: fetch.then(|| marketprice::Vecs::import(path, compressed).unwrap()), + blocks: blocks::Vecs::forced_import(path, compressed)?, + indexes: indexes::Vecs::forced_import(path, compressed)?, + transactions: transactions::Vecs::forced_import(path, compressed)?, + marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()), }) } @@ -40,6 +44,9 @@ impl Vecs { ) -> color_eyre::Result<()> { let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; + self.blocks + .compute(indexer, &mut self.indexes, &starting_indexes, exit)?; + self.transactions .compute(indexer, &mut self.indexes, &starting_indexes, exit)?; diff --git a/crates/brk_computer/src/storage/vecs/stats/generic.rs b/crates/brk_computer/src/storage/vecs/stats/generic.rs new file mode 100644 index 000000000..d80cc02c9 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/stats/generic.rs @@ -0,0 +1,396 @@ +use std::path::Path; + +use brk_exit::Exit; +use brk_vec::{AnyStorableVec, Compressed, Result, StoredIndex, Version}; + +use crate::storage::vecs::base::StorableVec; + +use super::StoredType; + +#[derive(Clone, Debug)] +pub struct StorableVecGeneator +where + I: StoredIndex, + T: StoredType, +{ + // first: Option>, + average: Option>, + sum: Option>, + max: Option>, + _90p: Option>, + _75p: Option>, + median: Option>, + _25p: Option>, + _10p: Option>, + min: Option>, + last: Option>, +} + +impl StorableVecGeneator +where + I: StoredIndex, + T: StoredType, +{ + pub fn forced_import( + path: &Path, + compressed: Compressed, + options: StorableVecGeneatorOptions, + ) -> color_eyre::Result { + let name = path.file_name().unwrap().to_str().unwrap().to_string(); + + // let prefix = |s: &str| path.with_file_name(format!("{s}_{name}")); + let suffix = |s: &str| path.with_file_name(format!("{name}_{s}")); + + let s = Self { + // first: options.first.then(|| { + // StorableVec::forced_import(&prefix("first"), Version::from(1), compressed).unwrap() + // }), + last: options + .last + .then(|| StorableVec::forced_import(path, Version::from(1), compressed).unwrap()), + min: options.min.then(|| { + StorableVec::forced_import(&suffix("min"), Version::from(1), compressed).unwrap() + }), + max: options.max.then(|| { + StorableVec::forced_import(&suffix("max"), Version::from(1), compressed).unwrap() + }), + median: options.median.then(|| { + StorableVec::forced_import(&suffix("median"), Version::from(1), compressed).unwrap() + }), + average: options.average.then(|| { + StorableVec::forced_import(&suffix("average"), Version::from(1), compressed) + .unwrap() + }), + sum: options.sum.then(|| { + StorableVec::forced_import(&suffix("sum"), Version::from(1), compressed).unwrap() + }), + _90p: options._90p.then(|| { + StorableVec::forced_import(&suffix("90p"), Version::from(1), compressed).unwrap() + }), + _75p: options._75p.then(|| { + StorableVec::forced_import(&suffix("75p"), Version::from(1), compressed).unwrap() + }), + _25p: options._25p.then(|| { + StorableVec::forced_import(&suffix("25p"), Version::from(1), compressed).unwrap() + }), + _10p: options._10p.then(|| { + StorableVec::forced_import(&suffix("10p"), Version::from(1), compressed).unwrap() + }), + }; + + Ok(s) + } + + pub fn compute( + &mut self, + max_from: I, + source: &mut StorableVec, + first_indexes: &mut brk_vec::StorableVec, + last_indexes: &mut brk_vec::StorableVec, + exit: &Exit, + ) -> Result<()> + where + I2: StoredIndex + StoredType, + T: Ord, + { + let index = self.starting_index(max_from); + + first_indexes.iter_from(index, |(i, first_index)| { + let first_index = *first_index; + let last_index = *last_indexes.get(i).unwrap().unwrap(); + + // if let Some(first) = self.first.as_mut() { + // let v = source.read(first_index).unwrap().unwrap(); + // first.forced_push_at(index, v.clone(), exit); + // } + + if let Some(last) = self.last.as_mut() { + let v = source.get(last_index).unwrap().unwrap(); + last.forced_push_at(index, v.clone(), exit)?; + } + + let first_index = first_index.to_usize()?; + let last_index = last_index.to_usize()?; + + let needs_sum_or_average = self.sum.is_some() || self.average.is_some(); + let needs_sorted = self.max.is_some() + || self._90p.is_some() + || self._75p.is_some() + || self.median.is_some() + || self._25p.is_some() + || self._10p.is_some() + || self.min.is_some(); + let needs_values = needs_sorted || needs_sum_or_average; + + if needs_values { + let mut values = + source.collect_range(Some(first_index as i64), Some(last_index as i64))?; + + if needs_sorted { + values.sort_unstable(); + + if let Some(max) = self.max.as_mut() { + max.forced_push_at(i, values.last().unwrap().clone(), exit)?; + } + + if let Some(_90p) = self._90p.as_mut() { + _90p.forced_push_at(i, Self::get_percentile(&values, 0.90), exit)?; + } + + if let Some(_75p) = self._75p.as_mut() { + _75p.forced_push_at(i, Self::get_percentile(&values, 0.75), exit)?; + } + + if let Some(median) = self.median.as_mut() { + median.forced_push_at(i, Self::get_percentile(&values, 0.50), exit)?; + } + + if let Some(_25p) = self._25p.as_mut() { + _25p.forced_push_at(i, Self::get_percentile(&values, 0.25), exit)?; + } + + if let Some(_10p) = self._10p.as_mut() { + _10p.forced_push_at(i, Self::get_percentile(&values, 0.10), exit)?; + } + + if let Some(min) = self.min.as_mut() { + min.forced_push_at(i, values.first().unwrap().clone(), exit)?; + } + } + + if needs_sum_or_average { + let len = values.len(); + + if let Some(average) = self.average.as_mut() { + let a = values + .iter() + .map(|v| v.clone() / len) + .fold(T::from(0), |a, b| a + b); + average.forced_push_at(i, a, exit)?; + } + + if let Some(sum_vec) = self.sum.as_mut() { + let sum = values.into_iter().fold(T::from(0), |a, b| a + b); + sum_vec.forced_push_at(i, sum, exit)?; + } + } + } + + Ok(()) + })?; + Ok(()) + } + + fn get_percentile(sorted: &[T], percentile: f64) -> T { + let len = sorted.len(); + + if len == 0 { + panic!(); + } else if len == 1 { + sorted[0].clone() + } else { + let index = (len - 1) as f64 * percentile; + + let fract = index.fract(); + + if fract != 0.0 { + let left = sorted.get(index as usize).unwrap().clone(); + let right = sorted.get(index.ceil() as usize).unwrap().clone(); + left / 2 + right / 2 + } else { + sorted.get(index as usize).unwrap().clone() + } + } + } + + fn starting_index(&self, max_from: I) -> I { + max_from.min(I::from( + self.as_any_vecs() + .into_iter() + .map(|v| v.len()) + .min() + .unwrap(), + )) + } + + pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + let mut v: Vec<&dyn AnyStorableVec> = vec![]; + + // if let Some(first) = self.first.as_ref() { + // v.push(first.any_vec()); + // } + if let Some(last) = self.last.as_ref() { + v.push(last.any_vec()); + } + if let Some(min) = self.min.as_ref() { + v.push(min.any_vec()); + } + if let Some(max) = self.max.as_ref() { + v.push(max.any_vec()); + } + if let Some(median) = self.median.as_ref() { + v.push(median.any_vec()); + } + if let Some(average) = self.average.as_ref() { + v.push(average.any_vec()); + } + if let Some(sum) = self.sum.as_ref() { + v.push(sum.any_vec()); + } + if let Some(_90p) = self._90p.as_ref() { + v.push(_90p.any_vec()); + } + if let Some(_75p) = self._75p.as_ref() { + v.push(_75p.any_vec()); + } + if let Some(_25p) = self._25p.as_ref() { + v.push(_25p.any_vec()); + } + if let Some(_10p) = self._10p.as_ref() { + v.push(_10p.any_vec()); + } + + v + } +} + +#[derive(Default, Clone, Copy)] +pub struct StorableVecGeneatorOptions { + average: bool, + sum: bool, + max: bool, + _90p: bool, + _75p: bool, + median: bool, + _25p: bool, + _10p: bool, + min: bool, + // first: bool, + last: bool, +} + +impl StorableVecGeneatorOptions { + // pub fn add_first(mut self) -> Self { + // self.first = true; + // self + // } + + pub fn add_last(mut self) -> Self { + self.last = true; + self + } + + pub fn add_min(mut self) -> Self { + self.min = true; + self + } + + pub fn add_max(mut self) -> Self { + self.max = true; + self + } + + pub fn add_median(mut self) -> Self { + self.median = true; + self + } + + pub fn add_average(mut self) -> Self { + self.average = true; + self + } + + pub fn add_sum(mut self) -> Self { + self.sum = true; + self + } + + pub fn add_90p(mut self) -> Self { + self._90p = true; + self + } + + pub fn add_75p(mut self) -> Self { + self._75p = true; + self + } + + pub fn add_25p(mut self) -> Self { + self._25p = true; + self + } + + pub fn add_10p(mut self) -> Self { + self._10p = true; + self + } + + pub fn rm_min(mut self) -> Self { + self.min = false; + self + } + + pub fn rm_max(mut self) -> Self { + self.max = false; + self + } + + pub fn rm_median(mut self) -> Self { + self.median = false; + self + } + + pub fn rm_average(mut self) -> Self { + self.average = false; + self + } + + pub fn rm_sum(mut self) -> Self { + self.sum = false; + self + } + + pub fn rm_90p(mut self) -> Self { + self._90p = false; + self + } + + pub fn rm_75p(mut self) -> Self { + self._75p = false; + self + } + + pub fn rm_25p(mut self) -> Self { + self._25p = false; + self + } + + pub fn rm_10p(mut self) -> Self { + self._10p = false; + self + } + + pub fn add_minmax(mut self) -> Self { + self.min = true; + self.max = true; + self + } + + pub fn add_percentiles(mut self) -> Self { + self._90p = true; + self._75p = true; + self.median = true; + self._25p = true; + self._10p = true; + self + } + + pub fn remove_percentiles(mut self) -> Self { + self._90p = false; + self._75p = false; + self.median = false; + self._25p = false; + self._10p = false; + self + } +} diff --git a/crates/brk_computer/src/storage/vecs/stats/grouped.rs b/crates/brk_computer/src/storage/vecs/stats/grouped.rs new file mode 100644 index 000000000..12876ff79 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/stats/grouped.rs @@ -0,0 +1,72 @@ +use std::path::Path; + +use brk_core::{ + Dateindex, Decadeindex, Difficultyepoch, Halvingepoch, Height, Monthindex, Weekindex, Yearindex, +}; +use brk_indexer::{Indexer, Indexes}; +use brk_vec::{AnyStorableVec, Compressed}; + +use crate::storage::vecs::{base::StorableVec, indexes}; + +use super::{StorableVecGeneator, StorableVecGeneatorOptions, StoredType}; + +#[derive(Clone)] +pub struct StorableVecGeneatorByIndex +where + T: StoredType, +{ + pub dateindex: StorableVecGeneator, + pub weekindex: StorableVecGeneator, + pub difficultyepoch: StorableVecGeneator, + pub monthindex: StorableVecGeneator, + pub yearindex: StorableVecGeneator, + pub halvingepoch: StorableVecGeneator, + pub decadeindex: StorableVecGeneator, +} + +impl StorableVecGeneatorByIndex +where + T: StoredType, +{ + pub fn forced_import( + path: &Path, + compressed: Compressed, + options: StorableVecGeneatorOptions, + ) -> color_eyre::Result { + let dateindex = StorableVecGeneator::forced_import(path, compressed, options)?; + + let options = options.remove_percentiles(); + + Ok(Self { + dateindex, + weekindex: StorableVecGeneator::forced_import(path, compressed, options)?, + difficultyepoch: StorableVecGeneator::forced_import(path, compressed, options)?, + monthindex: StorableVecGeneator::forced_import(path, compressed, options)?, + yearindex: StorableVecGeneator::forced_import(path, compressed, options)?, + halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?, + decadeindex: StorableVecGeneator::forced_import(path, compressed, options)?, + }) + } + + pub fn compute( + &mut self, + source: &mut StorableVec, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + starting_indexes: &Indexes, + ) { + } + + pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + [ + self.dateindex.as_any_vecs(), + self.weekindex.as_any_vecs(), + self.difficultyepoch.as_any_vecs(), + self.monthindex.as_any_vecs(), + self.yearindex.as_any_vecs(), + self.halvingepoch.as_any_vecs(), + self.decadeindex.as_any_vecs(), + ] + .concat() + } +} diff --git a/crates/brk_computer/src/storage/vecs/stats/mod.rs b/crates/brk_computer/src/storage/vecs/stats/mod.rs new file mode 100644 index 000000000..e3b3d25f7 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/stats/mod.rs @@ -0,0 +1,7 @@ +mod generic; +mod grouped; +mod stored_type; + +pub use generic::*; +pub use grouped::*; +pub use stored_type::*; diff --git a/crates/brk_computer/src/storage/vecs/stats/stored_type.rs b/crates/brk_computer/src/storage/vecs/stats/stored_type.rs new file mode 100644 index 000000000..9061d2c92 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/stats/stored_type.rs @@ -0,0 +1,11 @@ +use std::ops::{Add, Div}; + +pub trait StoredType +where + Self: brk_vec::StoredType + From + Div + Add, +{ +} +impl StoredType for T where + T: brk_vec::StoredType + From + Div + Add +{ +} diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index eab1b847c..3cbb7ccf0 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -9,7 +9,6 @@ use super::{Indexes, StorableVec, indexes}; #[derive(Clone)] pub struct Vecs { - // pub height_to_block_interval: StorableVec, // pub height_to_fee: StorableVec, // pub height_to_inputcount: StorableVec, // pub height_to_maxfeerate: StorableVec, @@ -22,24 +21,30 @@ pub struct Vecs { // pub txindex_to_fee: StorableVec, pub txindex_to_is_coinbase: StorableVec, // pub txindex_to_feerate: StorableVec, - // pub txindex_to_inputs_count: StorableVec, + pub txindex_to_inputs_count: StorableVec, // pub txindex_to_inputs_sum: StorableVec, - // pub txindex_to_outputs_count: StorableVec, + pub txindex_to_outputs_count: StorableVec, // pub txindex_to_outputs_sum: StorableVec, + // pub txinindex_to_value: StorableVec, } impl Vecs { - pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { - // height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?, // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?, - // height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?, + // height_to_input_count: StorableVec::forced_import( + // &path.join("height_to_input_count"), + // Version::from(1), + // )?, // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?, // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?, // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?, - // height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?, + // height_to_output_count: StorableVec::forced_import( + // &path.join("height_to_output_count"), + // Version::from(1), + // )?, // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?, // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?, // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?, @@ -47,28 +52,35 @@ impl Vecs { // &path.join("txindex_to_fee"), // Version::from(1), // )?, - txindex_to_is_coinbase: StorableVec::import( + txindex_to_is_coinbase: StorableVec::forced_import( &path.join("txindex_to_is_coinbase"), Version::from(1), compressed, )?, // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?, - // txindex_to_inputs_count: StorableVec::forced_import( - // &path.join("txindex_to_inputs_count"), - // Version::from(1), - // )?, + txindex_to_inputs_count: StorableVec::forced_import( + &path.join("txindex_to_inputs_count"), + Version::from(1), + compressed, + )?, // txindex_to_inputs_sum: StorableVec::forced_import( // &path.join("txindex_to_inputs_sum"), // Version::from(1), // )?, - // txindex_to_outputs_count: StorableVec::forced_import( - // &path.join("txindex_to_outputs_count"), - // Version::from(1), - // )?, + txindex_to_outputs_count: StorableVec::forced_import( + &path.join("txindex_to_outputs_count"), + Version::from(1), + compressed, + )?, // txindex_to_outputs_sum: StorableVec::forced_import( // &path.join("txindex_to_outputs_sum"), // Version::from(1), // )?, + // txinindex_to_value: StorableVec::forced_import( + // &path.join("txinindex_to_value"), + // Version::from(1), + // compressed, + // )?, }) } @@ -81,27 +93,43 @@ impl Vecs { ) -> color_eyre::Result<()> { let indexer_vecs = indexer.mut_vecs(); - // self.vecs.txindex_to_inputs_count.compute_count_from_indexes( - // starting_indexes.txindex, - // &mut indexer.vecs().txindex_to_first_txinindex, - // &mut self.vecs.txindex_to_last_txinindex, - // exit, - // )?; + self.txindex_to_inputs_count.compute_count_from_indexes( + starting_indexes.txindex, + indexer_vecs.txindex_to_first_txinindex.mut_vec(), + indexes.txindex_to_last_txinindex.mut_vec(), + exit, + )?; - // self.vecs.txindex_to_outputs_count.compute_count_from_indexes( - // starting_indexes.txindex, - // &mut indexer.vecs().txindex_to_first_txoutindex, - // &mut self.vecs.txindex_to_last_txoutindex, - // exit, - // )?; + self.txindex_to_outputs_count.compute_count_from_indexes( + starting_indexes.txindex, + indexer_vecs.txindex_to_first_txoutindex.mut_vec(), + indexes.txindex_to_last_txoutindex.mut_vec(), + exit, + )?; self.txindex_to_is_coinbase.compute_is_first_ordered( starting_indexes.txindex, - &mut indexer_vecs.txindex_to_height, - &mut indexer_vecs.height_to_first_txindex, + indexer_vecs.txindex_to_height.mut_vec(), + indexer_vecs.height_to_first_txindex.mut_vec(), exit, )?; + // self.txinindex_to_value.compute_transform( + // starting_indexes.txinindex, + // indexer_vecs.txinindex_to_txoutindex.mut_vec(), + // |(txinindex, txoutindex, slf, other)| { + // let value = + // if let Ok(Some(value)) = indexer_vecs.txoutindex_to_value.read(txoutindex) { + // *value + // } else { + // dbg!(txinindex, txoutindex, slf.len(), other.len()); + // panic!() + // }; + // (txinindex, value) + // }, + // exit, + // )?; + // self.vecs.txindex_to_fee.compute_transform( // &mut self.vecs.txindex_to_height, // &mut indexer.vecs().height_to_first_txindex, @@ -128,6 +156,10 @@ impl Vecs { } pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { - vec![&*self.txindex_to_is_coinbase] + vec![ + self.txindex_to_is_coinbase.any_vec(), + self.txindex_to_inputs_count.any_vec(), + self.txindex_to_outputs_count.any_vec(), + ] } } diff --git a/crates/brk_core/src/structs/decadeindex.rs b/crates/brk_core/src/structs/decadeindex.rs new file mode 100644 index 000000000..cb623f00b --- /dev/null +++ b/crates/brk_core/src/structs/decadeindex.rs @@ -0,0 +1,48 @@ +use std::{fmt::Debug, ops::Add}; + +use serde::{Deserialize, Serialize}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct Decadeindex(u8); + +impl From for Decadeindex { + fn from(value: u8) -> Self { + Self(value) + } +} + +impl From for Decadeindex { + fn from(value: usize) -> Self { + Self(value as u8) + } +} + +impl From for usize { + fn from(value: Decadeindex) -> Self { + value.0 as usize + } +} + +impl Add for Decadeindex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self::from(self.0 + rhs as u8) + } +} diff --git a/crates/brk_core/src/structs/difficultyepoch.rs b/crates/brk_core/src/structs/difficultyepoch.rs new file mode 100644 index 000000000..13d68339f --- /dev/null +++ b/crates/brk_core/src/structs/difficultyepoch.rs @@ -0,0 +1,48 @@ +use std::{fmt::Debug, ops::Add}; + +use serde::{Deserialize, Serialize}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct Difficultyepoch(u16); + +impl From for Difficultyepoch { + fn from(value: u16) -> Self { + Self(value) + } +} + +impl From for Difficultyepoch { + fn from(value: usize) -> Self { + Self(value as u16) + } +} + +impl From for usize { + fn from(value: Difficultyepoch) -> Self { + value.0 as usize + } +} + +impl Add for Difficultyepoch { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self::from(self.0 + rhs as u16) + } +} diff --git a/crates/brk_core/src/structs/halvingepoch.rs b/crates/brk_core/src/structs/halvingepoch.rs new file mode 100644 index 000000000..bebd5869c --- /dev/null +++ b/crates/brk_core/src/structs/halvingepoch.rs @@ -0,0 +1,48 @@ +use std::{fmt::Debug, ops::Add}; + +use serde::{Deserialize, Serialize}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct Halvingepoch(u8); + +impl From for Halvingepoch { + fn from(value: u8) -> Self { + Self(value) + } +} + +impl From for Halvingepoch { + fn from(value: usize) -> Self { + Self(value as u8) + } +} + +impl From for usize { + fn from(value: Halvingepoch) -> Self { + value.0 as usize + } +} + +impl Add for Halvingepoch { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self::from(self.0 + rhs as u8) + } +} diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index 17757ecc2..c4b85e8b7 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -67,7 +67,7 @@ impl PartialEq for Height { } impl Add for Height { - type Output = Height; + type Output = Self; fn add(self, rhs: Height) -> Self::Output { Self::from(self.0 + rhs.0) @@ -75,7 +75,7 @@ impl Add for Height { } impl Add for Height { - type Output = Height; + type Output = Self; fn add(self, rhs: u32) -> Self::Output { Self::from(self.0 + rhs) @@ -83,7 +83,7 @@ impl Add for Height { } impl Add for Height { - type Output = Height; + type Output = Self; fn add(self, rhs: usize) -> Self::Output { Self::from(self.0 + rhs as u32) @@ -91,8 +91,8 @@ impl Add for Height { } impl CheckedSub for Height { - fn checked_sub(self, rhs: Height) -> Option { - self.0.checked_sub(rhs.0).map(Height::from) + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self::from) } } @@ -115,14 +115,14 @@ impl AddAssign for Height { } impl Rem for Height { - type Output = Height; + type Output = Self; fn rem(self, rhs: Height) -> Self::Output { Self(self.0.rem(rhs.0)) } } impl Rem for Height { - type Output = Height; + type Output = Self; fn rem(self, rhs: usize) -> Self::Output { Self(self.0.rem(Height::from(rhs).0)) } diff --git a/crates/brk_core/src/structs/mod.rs b/crates/brk_core/src/structs/mod.rs index bbe35badf..cad8d3cb8 100644 --- a/crates/brk_core/src/structs/mod.rs +++ b/crates/brk_core/src/structs/mod.rs @@ -9,10 +9,14 @@ mod cents; mod compressed; mod date; mod dateindex; +mod decadeindex; +mod difficultyepoch; mod dollars; mod feerate; +mod halvingepoch; mod height; mod locktime; +mod monthindex; mod ohlc; mod sats; mod timestamp; @@ -24,7 +28,9 @@ mod txversion; mod unit; mod vin; mod vout; +mod weekindex; mod weight; +mod yearindex; pub use addressbytes::*; pub use addressindex::*; @@ -37,10 +43,14 @@ pub use cents::*; pub use compressed::*; pub use date::*; pub use dateindex::*; +pub use decadeindex::*; +pub use difficultyepoch::*; pub use dollars::*; pub use feerate::*; +pub use halvingepoch::*; pub use height::*; pub use locktime::*; +pub use monthindex::*; pub use ohlc::*; pub use sats::*; pub use timestamp::*; @@ -52,4 +62,6 @@ pub use txversion::*; pub use unit::*; pub use vin::*; pub use vout::*; +pub use weekindex::*; pub use weight::*; +pub use yearindex::*; diff --git a/crates/brk_core/src/structs/monthindex.rs b/crates/brk_core/src/structs/monthindex.rs new file mode 100644 index 000000000..28fc53130 --- /dev/null +++ b/crates/brk_core/src/structs/monthindex.rs @@ -0,0 +1,48 @@ +use std::{fmt::Debug, ops::Add}; + +use serde::{Deserialize, Serialize}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct Monthindex(u16); + +impl From for Monthindex { + fn from(value: u16) -> Self { + Self(value) + } +} + +impl From for Monthindex { + fn from(value: usize) -> Self { + Self(value as u16) + } +} + +impl From for usize { + fn from(value: Monthindex) -> Self { + value.0 as usize + } +} + +impl Add for Monthindex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self::from(self.0 + rhs as u16) + } +} diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs index cc07a62c5..d57571b36 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -38,7 +38,7 @@ impl Sats { } impl Add for Sats { - type Output = Sats; + type Output = Self; fn add(self, rhs: Sats) -> Self::Output { Sats::from(self.0 + rhs.0) } @@ -51,8 +51,8 @@ impl AddAssign for Sats { } impl CheckedSub for Sats { - fn checked_sub(self, rhs: Sats) -> Option { - self.0.checked_sub(rhs.0).map(Sats::from) + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self::from) } } @@ -63,21 +63,21 @@ impl SubAssign for Sats { } impl Mul for Sats { - type Output = Sats; + type Output = Self; fn mul(self, rhs: Sats) -> Self::Output { Sats::from(self.0 * rhs.0) } } impl Mul for Sats { - type Output = Sats; + type Output = Self; fn mul(self, rhs: u64) -> Self::Output { Sats::from(self.0 * rhs) } } impl Mul for Sats { - type Output = Sats; + type Output = Self; fn mul(self, rhs: Height) -> Self::Output { Sats::from(self.0 * u64::from(rhs)) } diff --git a/crates/brk_core/src/structs/timestamp.rs b/crates/brk_core/src/structs/timestamp.rs index 605fbca49..77dd493de 100644 --- a/crates/brk_core/src/structs/timestamp.rs +++ b/crates/brk_core/src/structs/timestamp.rs @@ -1,14 +1,36 @@ +use std::ops::{Add, Div}; + use derive_deref::Deref; use jiff::{civil::date, tz::TimeZone}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + #[derive( - Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize, + Debug, + Deref, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, )] pub struct Timestamp(u32); impl Timestamp { + pub const ZERO: Self = Self(0); + + pub fn new(timestamp: u32) -> Self { + Self(timestamp) + } + pub fn floor_seconds(self) -> Self { let t = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC); let d = jiff::civil::DateTime::from(t); @@ -46,3 +68,29 @@ impl From for bitcoin::locktime::absolute::Time { bitcoin::locktime::absolute::Time::from_consensus(*value).unwrap() } } + +impl From for Timestamp { + fn from(value: usize) -> Self { + Self(value as u32) + } +} + +impl CheckedSub for Timestamp { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self::from) + } +} + +impl Div for Timestamp { + type Output = Self; + fn div(self, rhs: usize) -> Self::Output { + Self(self.0 / rhs as u32) + } +} + +impl Add for Timestamp { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} diff --git a/crates/brk_core/src/structs/txinindex.rs b/crates/brk_core/src/structs/txinindex.rs index 5ab3d8ee3..04544d593 100644 --- a/crates/brk_core/src/structs/txinindex.rs +++ b/crates/brk_core/src/structs/txinindex.rs @@ -61,8 +61,8 @@ impl AddAssign for Txinindex { } impl CheckedSub for Txinindex { - fn checked_sub(self, rhs: Txinindex) -> Option { - self.0.checked_sub(rhs.0).map(Txinindex::from) + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self::from) } } diff --git a/crates/brk_core/src/structs/txoutindex.rs b/crates/brk_core/src/structs/txoutindex.rs index ca58888cc..ba4571505 100644 --- a/crates/brk_core/src/structs/txoutindex.rs +++ b/crates/brk_core/src/structs/txoutindex.rs @@ -67,8 +67,8 @@ impl AddAssign for Txoutindex { } impl CheckedSub for Txoutindex { - fn checked_sub(self, rhs: Txoutindex) -> Option { - self.0.checked_sub(rhs.0).map(Txoutindex::from) + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self::from) } } diff --git a/crates/brk_core/src/structs/weekindex.rs b/crates/brk_core/src/structs/weekindex.rs new file mode 100644 index 000000000..1f3ef677d --- /dev/null +++ b/crates/brk_core/src/structs/weekindex.rs @@ -0,0 +1,48 @@ +use std::{fmt::Debug, ops::Add}; + +use serde::{Deserialize, Serialize}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct Weekindex(u16); + +impl From for Weekindex { + fn from(value: u16) -> Self { + Self(value) + } +} + +impl From for Weekindex { + fn from(value: usize) -> Self { + Self(value as u16) + } +} + +impl From for usize { + fn from(value: Weekindex) -> Self { + value.0 as usize + } +} + +impl Add for Weekindex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self::from(self.0 + rhs as u16) + } +} diff --git a/crates/brk_core/src/structs/yearindex.rs b/crates/brk_core/src/structs/yearindex.rs new file mode 100644 index 000000000..6da0584d9 --- /dev/null +++ b/crates/brk_core/src/structs/yearindex.rs @@ -0,0 +1,48 @@ +use std::{fmt::Debug, ops::Add}; + +use serde::{Deserialize, Serialize}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct Yearindex(u8); + +impl From for Yearindex { + fn from(value: u8) -> Self { + Self(value) + } +} + +impl From for Yearindex { + fn from(value: usize) -> Self { + Self(value as u8) + } +} + +impl From for usize { + fn from(value: Yearindex) -> Self { + value.0 as usize + } +} + +impl Add for Yearindex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self::from(self.0 + rhs as u8) + } +} diff --git a/crates/brk_fetcher/examples/main.rs b/crates/brk_fetcher/examples/main.rs index e7ee777ea..c06e21ed5 100644 --- a/crates/brk_fetcher/examples/main.rs +++ b/crates/brk_fetcher/examples/main.rs @@ -11,8 +11,8 @@ fn main() -> color_eyre::Result<()> { dbg!(fetcher.get_date(Date::new(2025, 1, 1))?); dbg!(fetcher.get_height( 881_000_u32.into(), - 1740683986.into(), - Some(1740683000.into()) + 1740683986_u32.into(), + Some(1740683000_u32.into()) )?); Ok(()) diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index 3b192b79f..eb158b9cd 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -108,7 +108,7 @@ impl Fetcher { let mut final_ohlc = OHLCCents::from(previous_ohlc.close); - let start = previous_timestamp.unwrap_or(Timestamp::from(0)); + let start = previous_timestamp.unwrap_or(Timestamp::new(0)); let end = timestamp; // Otherwise it's a re-org diff --git a/crates/brk_indexer/examples/main.rs b/crates/brk_indexer/examples/main.rs index 07c3ef63c..864845afa 100644 --- a/crates/brk_indexer/examples/main.rs +++ b/crates/brk_indexer/examples/main.rs @@ -1,12 +1,9 @@ use std::path::Path; -use brk_core::{default_bitcoin_path, dot_brk_path}; +use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_parser::{ - Parser, - rpc::{self}, -}; +use brk_parser::{Parser, rpc}; fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 78fa7af29..3eb318877 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -7,7 +7,7 @@ use std::{ collections::BTreeMap, path::{Path, PathBuf}, str::FromStr, - thread::{self}, + thread, }; use brk_core::{ @@ -31,6 +31,7 @@ pub use stores::*; pub use vecs::*; const SNAPSHOT_BLOCK_RANGE: usize = 1000; +const COLLISIONS_CHECKED_UP_TO: u32 = 870_000; #[derive(Clone)] pub struct Indexer { @@ -138,7 +139,7 @@ impl Indexer { idxs.height = height; // Used to check rapidhash collisions - let check_collisions = self.check_collisions && height > Height::new(640_000); + let check_collisions = self.check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO); let blockhash = BlockHash::from(blockhash); let blockhash_prefix = BlockHashPrefix::from(&blockhash); diff --git a/crates/brk_indexer/src/vecs/base.rs b/crates/brk_indexer/src/vecs/base.rs index f6e8d5198..0b7f25e31 100644 --- a/crates/brk_indexer/src/vecs/base.rs +++ b/crates/brk_indexer/src/vecs/base.rs @@ -1,11 +1,14 @@ use std::{ + cmp::Ordering, fmt::Debug, io, - ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; -use brk_vec::{Compressed, StoredIndex, StoredType, Version}; +use brk_vec::{ + AnyStorableVec, Compressed, Error, MAX_CACHE_SIZE, MAX_PAGE_SIZE, Result, StoredIndex, + StoredType, Value, Version, +}; use super::Height; @@ -20,7 +23,16 @@ where I: StoredIndex, T: StoredType, { - pub fn import(path: &Path, version: Version, compressed: Compressed) -> brk_vec::Result { + pub const SIZE_OF_T: usize = size_of::(); + pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; + pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T; + pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE; + + pub fn forced_import( + path: &Path, + version: Version, + compressed: Compressed, + ) -> brk_vec::Result { let mut vec = brk_vec::StorableVec::forced_import(path, version, compressed)?; vec.enable_large_cache(); @@ -31,6 +43,76 @@ where }) } + #[inline] + pub fn get(&self, index: I) -> Result>> { + self.get_(index.to_usize()?) + } + fn get_(&self, index: usize) -> Result>> { + match self.vec.index_to_pushed_index(index) { + Ok(index) => { + if let Some(index) = index { + return Ok(self.vec.pushed().get(index).map(|v| Value::Ref(v))); + } + } + Err(Error::IndexTooHigh) => return Ok(None), + Err(Error::IndexTooLow) => {} + Err(error) => return Err(error), + } + + let large_cache_len = self.vec.large_cache_len(); + if large_cache_len != 0 { + let page_index = Self::index_to_page_index(index); + let last_index = self.vec.stored_len() - 1; + let max_page_index = Self::index_to_page_index(last_index); + let min_page_index = (max_page_index + 1) - large_cache_len; + + if page_index >= min_page_index { + let values = self + .vec + .pages() + .unwrap() + .get(page_index - min_page_index) + .ok_or(Error::MmapsVecIsTooSmall)? + .get_or_init(|| self.vec.decode_page(page_index).unwrap()); + + return Ok(values.get(index)?.map(|v| Value::Ref(v))); + } + } + + Ok(self.vec.read_(index)?.map(|v| Value::Owned(v))) + } + + pub fn iter_from(&mut self, index: I, f: F) -> Result<()> + where + F: FnMut((I, &T)) -> Result<()>, + { + self.vec.iter_from(index, f) + } + + #[inline(always)] + fn index_to_page_index(index: usize) -> usize { + index / Self::PER_PAGE + } + + #[inline] + pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { + match self.vec.len().cmp(&index.to_usize()?) { + Ordering::Greater => { + // dbg!(len, index, &self.pathbuf); + // panic!(); + Ok(()) + } + Ordering::Equal => { + self.vec.push(value); + Ok(()) + } + Ordering::Less => { + dbg!(index, value); + Err(Error::IndexTooHigh) + } + } + } + pub fn truncate_if_needed(&mut self, index: I, height: Height) -> brk_vec::Result<()> { if self.height.is_none_or(|self_height| self_height != height) { height.write(&self.path_height())?; @@ -39,6 +121,36 @@ where Ok(()) } + pub fn flush(&mut self, height: Height) -> io::Result<()> { + height.write(&self.path_height())?; + self.vec.flush() + } + + pub fn vec(&self) -> &brk_vec::StorableVec { + &self.vec + } + + pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec { + &mut self.vec + } + + pub fn any_vec(&self) -> &dyn AnyStorableVec { + &self.vec + } + + pub fn len(&self) -> usize { + self.vec.len() + } + + pub fn is_empty(&self) -> bool { + self.vec.is_empty() + } + + #[inline] + pub fn hasnt(&self, index: I) -> Result { + self.vec.has(index).map(|b| !b) + } + pub fn height(&self) -> brk_core::Result { Height::try_from(self.path_height().as_path()) } @@ -48,24 +160,19 @@ where fn path_height_(path: &Path) -> PathBuf { path.join("height") } - - pub fn flush(&mut self, height: Height) -> io::Result<()> { - height.write(&self.path_height())?; - self.vec.flush() - } } -impl Deref for StorableVec { - type Target = brk_vec::StorableVec; - fn deref(&self) -> &Self::Target { - &self.vec - } -} -impl DerefMut for StorableVec { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.vec - } -} +// impl Deref for StorableVec { +// type Target = brk_vec::StorableVec; +// fn deref(&self) -> &Self::Target { +// &self.vec +// } +// } +// impl DerefMut for StorableVec { +// fn deref_mut(&mut self) -> &mut Self::Target { +// &mut self.vec +// } +// } impl Clone for StorableVec where I: StoredIndex, diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 1bf3c813f..7f32f6f6e 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -58,6 +58,7 @@ pub struct Vecs { pub txindex_to_total_size: StorableVec, pub txindex_to_is_explicitly_rbf: StorableVec, pub txindex_to_txversion: StorableVec, + /// If txoutindex == Txoutindex MAX then is it's coinbase pub txinindex_to_txoutindex: StorableVec, pub txoutindex_to_addressindex: StorableVec, pub txoutindex_to_value: StorableVec, @@ -68,217 +69,217 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - addressindex_to_addresstype: StorableVec::import( + addressindex_to_addresstype: StorableVec::forced_import( &path.join("addressindex_to_addresstype"), Version::from(1), compressed, )?, - addressindex_to_addresstypeindex: StorableVec::import( + addressindex_to_addresstypeindex: StorableVec::forced_import( &path.join("addressindex_to_addresstypeindex"), Version::from(1), compressed, )?, - addressindex_to_height: StorableVec::import( + addressindex_to_height: StorableVec::forced_import( &path.join("addressindex_to_height"), Version::from(1), compressed, )?, - height_to_blockhash: StorableVec::import( + height_to_blockhash: StorableVec::forced_import( &path.join("height_to_blockhash"), Version::from(1), Compressed::NO, )?, - height_to_difficulty: StorableVec::import( + height_to_difficulty: StorableVec::forced_import( &path.join("height_to_difficulty"), Version::from(1), compressed, )?, - height_to_first_addressindex: StorableVec::import( + height_to_first_addressindex: StorableVec::forced_import( &path.join("height_to_first_addressindex"), Version::from(1), compressed, )?, - height_to_first_emptyindex: StorableVec::import( + height_to_first_emptyindex: StorableVec::forced_import( &path.join("height_to_first_emptyindex"), Version::from(1), compressed, )?, - height_to_first_multisigindex: StorableVec::import( + height_to_first_multisigindex: StorableVec::forced_import( &path.join("height_to_first_multisigindex"), Version::from(1), compressed, )?, - height_to_first_opreturnindex: StorableVec::import( + height_to_first_opreturnindex: StorableVec::forced_import( &path.join("height_to_first_opreturnindex"), Version::from(1), compressed, )?, - height_to_first_pushonlyindex: StorableVec::import( + height_to_first_pushonlyindex: StorableVec::forced_import( &path.join("height_to_first_pushonlyindex"), Version::from(1), compressed, )?, - height_to_first_txindex: StorableVec::import( + height_to_first_txindex: StorableVec::forced_import( &path.join("height_to_first_txindex"), Version::from(1), compressed, )?, - height_to_first_txinindex: StorableVec::import( + height_to_first_txinindex: StorableVec::forced_import( &path.join("height_to_first_txinindex"), Version::from(1), compressed, )?, - height_to_first_txoutindex: StorableVec::import( + height_to_first_txoutindex: StorableVec::forced_import( &path.join("height_to_first_txoutindex"), Version::from(1), compressed, )?, - height_to_first_unknownindex: StorableVec::import( + height_to_first_unknownindex: StorableVec::forced_import( &path.join("height_to_first_unkownindex"), Version::from(1), compressed, )?, - height_to_first_p2pk33index: StorableVec::import( + height_to_first_p2pk33index: StorableVec::forced_import( &path.join("height_to_first_p2pk33index"), Version::from(1), compressed, )?, - height_to_first_p2pk65index: StorableVec::import( + height_to_first_p2pk65index: StorableVec::forced_import( &path.join("height_to_first_p2pk65index"), Version::from(1), compressed, )?, - height_to_first_p2pkhindex: StorableVec::import( + height_to_first_p2pkhindex: StorableVec::forced_import( &path.join("height_to_first_p2pkhindex"), Version::from(1), compressed, )?, - height_to_first_p2shindex: StorableVec::import( + height_to_first_p2shindex: StorableVec::forced_import( &path.join("height_to_first_p2shindex"), Version::from(1), compressed, )?, - height_to_first_p2trindex: StorableVec::import( + height_to_first_p2trindex: StorableVec::forced_import( &path.join("height_to_first_p2trindex"), Version::from(1), compressed, )?, - height_to_first_p2wpkhindex: StorableVec::import( + height_to_first_p2wpkhindex: StorableVec::forced_import( &path.join("height_to_first_p2wpkhindex"), Version::from(1), compressed, )?, - height_to_first_p2wshindex: StorableVec::import( + height_to_first_p2wshindex: StorableVec::forced_import( &path.join("height_to_first_p2wshindex"), Version::from(1), compressed, )?, - height_to_size: StorableVec::import( + height_to_size: StorableVec::forced_import( &path.join("height_to_size"), Version::from(1), compressed, )?, - height_to_timestamp: StorableVec::import( + height_to_timestamp: StorableVec::forced_import( &path.join("height_to_timestamp"), Version::from(1), compressed, )?, - height_to_weight: StorableVec::import( + height_to_weight: StorableVec::forced_import( &path.join("height_to_weight"), Version::from(1), compressed, )?, - p2pk33index_to_p2pk33addressbytes: StorableVec::import( + p2pk33index_to_p2pk33addressbytes: StorableVec::forced_import( &path.join("p2pk33index_to_p2pk33addressbytes"), Version::from(1), Compressed::NO, )?, - p2pk65index_to_p2pk65addressbytes: StorableVec::import( + p2pk65index_to_p2pk65addressbytes: StorableVec::forced_import( &path.join("p2pk65index_to_p2pk65addressbytes"), Version::from(1), Compressed::NO, )?, - p2pkhindex_to_p2pkhaddressbytes: StorableVec::import( + p2pkhindex_to_p2pkhaddressbytes: StorableVec::forced_import( &path.join("p2pkhindex_to_p2pkhaddressbytes"), Version::from(1), Compressed::NO, )?, - p2shindex_to_p2shaddressbytes: StorableVec::import( + p2shindex_to_p2shaddressbytes: StorableVec::forced_import( &path.join("p2shindex_to_p2shaddressbytes"), Version::from(1), Compressed::NO, )?, - p2trindex_to_p2traddressbytes: StorableVec::import( + p2trindex_to_p2traddressbytes: StorableVec::forced_import( &path.join("p2trindex_to_p2traddressbytes"), Version::from(1), Compressed::NO, )?, - p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import( + p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::forced_import( &path.join("p2wpkhindex_to_p2wpkhaddressbytes"), Version::from(1), Compressed::NO, )?, - p2wshindex_to_p2wshaddressbytes: StorableVec::import( + p2wshindex_to_p2wshaddressbytes: StorableVec::forced_import( &path.join("p2wshindex_to_p2wshaddressbytes"), Version::from(1), Compressed::NO, )?, - txindex_to_first_txinindex: StorableVec::import( + txindex_to_first_txinindex: StorableVec::forced_import( &path.join("txindex_to_first_txinindex"), Version::from(1), compressed, )?, - txindex_to_first_txoutindex: StorableVec::import( + txindex_to_first_txoutindex: StorableVec::forced_import( &path.join("txindex_to_first_txoutindex"), Version::from(1), Compressed::NO, )?, - txindex_to_height: StorableVec::import( + txindex_to_height: StorableVec::forced_import( &path.join("txindex_to_height"), Version::from(1), compressed, )?, - txindex_to_locktime: StorableVec::import( + txindex_to_locktime: StorableVec::forced_import( &path.join("txindex_to_locktime"), Version::from(1), compressed, )?, - txindex_to_txid: StorableVec::import( + txindex_to_txid: StorableVec::forced_import( &path.join("txindex_to_txid"), Version::from(1), Compressed::NO, )?, - txindex_to_base_size: StorableVec::import( + txindex_to_base_size: StorableVec::forced_import( &path.join("txindex_to_base_size"), Version::from(1), compressed, )?, - txindex_to_total_size: StorableVec::import( + txindex_to_total_size: StorableVec::forced_import( &path.join("txindex_to_total_size"), Version::from(1), compressed, )?, - txindex_to_is_explicitly_rbf: StorableVec::import( + txindex_to_is_explicitly_rbf: StorableVec::forced_import( &path.join("txindex_to_is_explicitly_rbf"), Version::from(1), compressed, )?, - txindex_to_txversion: StorableVec::import( + txindex_to_txversion: StorableVec::forced_import( &path.join("txindex_to_txversion"), Version::from(1), compressed, )?, - txinindex_to_txoutindex: StorableVec::import( + txinindex_to_txoutindex: StorableVec::forced_import( &path.join("txinindex_to_txoutindex"), Version::from(1), compressed, )?, - txoutindex_to_addressindex: StorableVec::import( + txoutindex_to_addressindex: StorableVec::forced_import( &path.join("txoutindex_to_addressindex"), Version::from(1), compressed, )?, - txoutindex_to_value: StorableVec::import( + txoutindex_to_value: StorableVec::forced_import( &path.join("txoutindex_to_value"), Version::from(1), compressed, @@ -496,49 +497,49 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { vec![ - &*self.addressindex_to_addresstype, - &*self.addressindex_to_addresstypeindex, - &*self.addressindex_to_height, - &*self.height_to_blockhash, - &*self.height_to_difficulty, - &*self.height_to_first_addressindex, - &*self.height_to_first_emptyindex, - &*self.height_to_first_multisigindex, - &*self.height_to_first_opreturnindex, - &*self.height_to_first_pushonlyindex, - &*self.height_to_first_txindex, - &*self.height_to_first_txinindex, - &*self.height_to_first_txoutindex, - &*self.height_to_first_unknownindex, - &*self.height_to_first_p2pk33index, - &*self.height_to_first_p2pk65index, - &*self.height_to_first_p2pkhindex, - &*self.height_to_first_p2shindex, - &*self.height_to_first_p2trindex, - &*self.height_to_first_p2wpkhindex, - &*self.height_to_first_p2wshindex, - &*self.height_to_size, - &*self.height_to_timestamp, - &*self.height_to_weight, - &*self.p2pk33index_to_p2pk33addressbytes, - &*self.p2pk65index_to_p2pk65addressbytes, - &*self.p2pkhindex_to_p2pkhaddressbytes, - &*self.p2shindex_to_p2shaddressbytes, - &*self.p2trindex_to_p2traddressbytes, - &*self.p2wpkhindex_to_p2wpkhaddressbytes, - &*self.p2wshindex_to_p2wshaddressbytes, - &*self.txindex_to_first_txinindex, - &*self.txindex_to_first_txoutindex, - &*self.txindex_to_height, - &*self.txindex_to_locktime, - &*self.txindex_to_txid, - &*self.txindex_to_base_size, - &*self.txindex_to_total_size, - &*self.txindex_to_is_explicitly_rbf, - &*self.txindex_to_txversion, - &*self.txinindex_to_txoutindex, - &*self.txoutindex_to_addressindex, - &*self.txoutindex_to_value, + self.addressindex_to_addresstype.any_vec(), + self.addressindex_to_addresstypeindex.any_vec(), + self.addressindex_to_height.any_vec(), + self.height_to_blockhash.any_vec(), + self.height_to_difficulty.any_vec(), + self.height_to_first_addressindex.any_vec(), + self.height_to_first_emptyindex.any_vec(), + self.height_to_first_multisigindex.any_vec(), + self.height_to_first_opreturnindex.any_vec(), + self.height_to_first_pushonlyindex.any_vec(), + self.height_to_first_txindex.any_vec(), + self.height_to_first_txinindex.any_vec(), + self.height_to_first_txoutindex.any_vec(), + self.height_to_first_unknownindex.any_vec(), + self.height_to_first_p2pk33index.any_vec(), + self.height_to_first_p2pk65index.any_vec(), + self.height_to_first_p2pkhindex.any_vec(), + self.height_to_first_p2shindex.any_vec(), + self.height_to_first_p2trindex.any_vec(), + self.height_to_first_p2wpkhindex.any_vec(), + self.height_to_first_p2wshindex.any_vec(), + self.height_to_size.any_vec(), + self.height_to_timestamp.any_vec(), + self.height_to_weight.any_vec(), + self.p2pk33index_to_p2pk33addressbytes.any_vec(), + self.p2pk65index_to_p2pk65addressbytes.any_vec(), + self.p2pkhindex_to_p2pkhaddressbytes.any_vec(), + self.p2shindex_to_p2shaddressbytes.any_vec(), + self.p2trindex_to_p2traddressbytes.any_vec(), + self.p2wpkhindex_to_p2wpkhaddressbytes.any_vec(), + self.p2wshindex_to_p2wshaddressbytes.any_vec(), + self.txindex_to_first_txinindex.any_vec(), + self.txindex_to_first_txoutindex.any_vec(), + self.txindex_to_height.any_vec(), + self.txindex_to_locktime.any_vec(), + self.txindex_to_txid.any_vec(), + self.txindex_to_base_size.any_vec(), + self.txindex_to_total_size.any_vec(), + self.txindex_to_is_explicitly_rbf.any_vec(), + self.txindex_to_txversion.any_vec(), + self.txinindex_to_txoutindex.any_vec(), + self.txoutindex_to_addressindex.any_vec(), + self.txoutindex_to_value.any_vec(), ] } diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index 49eaf3691..432cc1444 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -3,14 +3,7 @@ #![doc = include_str!("../examples/main.rs")] #![doc = "```"] -use std::{ - cmp::Ordering, - collections::BTreeMap, - fs::{self}, - ops::ControlFlow, - path::PathBuf, - thread, -}; +use std::{cmp::Ordering, collections::BTreeMap, fs, ops::ControlFlow, path::PathBuf, thread}; use bitcoin::{Block, BlockHash}; use bitcoincore_rpc::RpcApi; diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index e6552877f..9dad5dce8 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -21,8 +21,8 @@ color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -oxc = { version = "0.58.1", features = ["codegen", "minifier"] } +oxc = { version = "0.60.0", features = ["codegen", "minifier"] } serde = { workspace = true } tokio = { version = "1.44.1", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full"] } -zip = "2.2.3" +zip = "2.4.1" diff --git a/crates/brk_server/src/files/file.rs b/crates/brk_server/src/files/file.rs index 0b0068738..e9cd998b6 100644 --- a/crates/brk_server/src/files/file.rs +++ b/crates/brk_server/src/files/file.rs @@ -1,8 +1,4 @@ -use std::{ - fs::{self}, - path::Path, - time::Instant, -}; +use std::{fs, path::Path, time::Instant}; use axum::{ body::Body, diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index 660d95013..d921361ef 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -24,12 +24,12 @@ fn main() -> Result<(), Box> { StorableVec::forced_import(Path::new("./vec"), Version::from(1), Compressed::YES)?; dbg!(vec.get(0)?); - dbg!(vec.read(0)?); - dbg!(vec.read(1)?); - dbg!(vec.read(2)?); - dbg!(vec.read(20)?); + dbg!(vec.get(0)?); + dbg!(vec.get(1)?); + dbg!(vec.get(2)?); dbg!(vec.get(20)?); - dbg!(vec.read(0)?); + dbg!(vec.get(20)?); + dbg!(vec.get(0)?); vec.push(21); vec.push(22); diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index eb9eda45c..63bfb1432 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -4,7 +4,6 @@ #![doc = "```"] use std::{ - cmp::Ordering, fs::{self, File, OpenOptions}, io::{self, Read, Seek, SeekFrom, Write}, marker::PhantomData, @@ -29,7 +28,7 @@ pub use traits::*; const ONE_KIB: usize = 1024; pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB; const ONE_MIB: usize = ONE_KIB * ONE_KIB; -const MAX_CACHE_SIZE: usize = 100 * ONE_MIB; +pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB; #[allow(private_interfaces)] #[derive(Debug)] @@ -83,49 +82,11 @@ where } #[inline] - pub fn get(&self, index: I) -> Result>> { - self.get_(Self::i_to_usize(index)?) - } - fn get_(&self, index: usize) -> Result>> { - match self.index_to_pushed_index(index) { - Ok(index) => { - if let Some(index) = index { - return Ok(self.pushed().get(index).map(|v| Value::Ref(v))); - } - } - Err(Error::IndexTooHigh) => return Ok(None), - Err(Error::IndexTooLow) => {} - Err(error) => return Err(error), - } - - let large_cache_len = self.large_cache_len(); - if large_cache_len != 0 { - let page_index = Self::index_to_page_index(index); - let last_index = self.stored_len() - 1; - let max_page_index = Self::index_to_page_index(last_index); - let min_page_index = (max_page_index + 1) - large_cache_len; - - if page_index >= min_page_index { - let values = self - .pages() - .unwrap() - .get(page_index - min_page_index) - .ok_or(Error::MmapsVecIsTooSmall)? - .get_or_init(|| self.decode_page(page_index).unwrap()); - - return Ok(values.get(index)?.map(|v| Value::Ref(v))); - } - } - - Ok(self.open_then_read_(index)?.map(|v| Value::Owned(v))) - } - - #[inline] - pub fn read(&mut self, index: I) -> Result> { - self.read_(Self::i_to_usize(index)?) + pub fn get(&mut self, index: I) -> Result> { + self.get_(index.to_usize()?) } #[inline] - pub fn read_(&mut self, index: usize) -> Result> { + pub fn get_(&mut self, index: usize) -> Result> { match self.index_to_pushed_index(index) { Ok(index) => { if let Some(index) = index { @@ -147,18 +108,18 @@ where self.page().unwrap().1.get(index) } - pub fn read_last(&mut self) -> Result> { + pub fn get_last(&mut self) -> Result> { let len = self.len(); if len == 0 { return Ok(None); } - self.read_(len - 1) + self.get_(len - 1) } - pub fn open_then_read(&self, index: I) -> Result> { - self.open_then_read_(Self::i_to_usize(index)?) + pub fn read(&self, index: I) -> Result> { + self.read_(index.to_usize()?) } - fn open_then_read_(&self, index: usize) -> Result> { + pub fn read_(&self, index: usize) -> Result> { Ok(match self { Self::Raw { .. } => { let mut file = self.open_file()?; @@ -193,7 +154,7 @@ where let stored_len = I::from(self.stored_len()); while index < stored_len { - let v = self.read(index)?.unwrap(); + let v = self.get(index)?.unwrap(); f((index, v))?; index = index + 1; } @@ -212,7 +173,7 @@ where let stored_len = I::from(self.stored_len()); while index < stored_len { - let v = self.read(index)?.unwrap().clone(); + let v = self.get(index)?.unwrap().clone(); f((index, v, self))?; index = index + 1; } @@ -265,7 +226,7 @@ where Ok(values) } - fn decode_page(&self, page_index: usize) -> Result> { + pub fn decode_page(&self, page_index: usize) -> Result> { Self::decode_page_( self.stored_len(), page_index, @@ -327,26 +288,7 @@ where #[inline] pub fn push(&mut self, value: T) { - self.mut_pushed().push(value) - } - - #[inline] - pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { - match self.len().cmp(&Self::i_to_usize(index)?) { - Ordering::Greater => { - // dbg!(len, index, &self.pathbuf); - // panic!(); - Ok(()) - } - Ordering::Equal => { - self.mut_pushed().push(value); - Ok(()) - } - Ordering::Less => { - dbg!(index, value); - Err(Error::IndexTooHigh) - } - } + self.mut_base().pushed.push(value) } pub fn flush(&mut self) -> io::Result<()> { @@ -464,7 +406,7 @@ where } pub fn truncate_if_needed(&mut self, index: I) -> Result<()> { - let index = Self::i_to_usize(index)?; + let index = index.to_usize()?; if index >= self.stored_len() { return Ok(()); @@ -580,7 +522,7 @@ where } } - fn large_cache_len(&self) -> usize { + pub fn large_cache_len(&self) -> usize { self.pages().map_or(0, |v| v.len()) } @@ -601,12 +543,7 @@ where } #[inline] - pub fn i_to_usize(index: I) -> Result { - index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize) - } - - #[inline] - fn index_to_pushed_index(&self, index: usize) -> Result> { + pub fn index_to_pushed_index(&self, index: usize) -> Result> { let stored_len = self.stored_len(); if index >= stored_len { @@ -662,7 +599,7 @@ where } #[inline] - fn pages(&self) -> Option<&Vec>>> { + pub fn pages(&self) -> Option<&Vec>>> { self.base().pages.as_ref() } @@ -682,7 +619,7 @@ where #[inline] pub fn has(&self, index: I) -> Result { - Ok(self.has_(Self::i_to_usize(index)?)) + Ok(self.has_(index.to_usize()?)) } #[inline] fn has_(&self, index: usize) -> bool { @@ -690,20 +627,10 @@ where } #[inline] - pub fn hasnt(&self, index: I) -> Result { - self.has(index).map(|b| !b) - } - - #[inline] - fn pushed(&self) -> &Vec { + pub fn pushed(&self) -> &Vec { &self.base().pushed } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - &mut self.mut_base().pushed - } - #[inline] pub fn pushed_len(&self) -> usize { self.pushed().len() @@ -715,7 +642,7 @@ where } #[inline] - fn stored_len(&self) -> usize { + pub fn stored_len(&self) -> usize { *self.base().stored_len } diff --git a/crates/brk_vec/src/structs/compressed.rs b/crates/brk_vec/src/structs/compressed.rs index 10bcb5fe5..3e35d7555 100644 --- a/crates/brk_vec/src/structs/compressed.rs +++ b/crates/brk_vec/src/structs/compressed.rs @@ -1,9 +1,4 @@ -use std::{ - fs, - io::{self}, - ops::Deref, - path::Path, -}; +use std::{fs, io, ops::Deref, path::Path}; use crate::{Error, Result}; diff --git a/crates/brk_vec/src/structs/version.rs b/crates/brk_vec/src/structs/version.rs index e356e94cd..9066a8ad8 100644 --- a/crates/brk_vec/src/structs/version.rs +++ b/crates/brk_vec/src/structs/version.rs @@ -9,7 +9,9 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Error, Result}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout)] +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout, +)] pub struct Version(u32); impl Version { @@ -54,7 +56,7 @@ impl TryFrom<&Path> for Version { } impl Add for Version { - type Output = Version; + type Output = Self; fn add(self, rhs: Version) -> Self::Output { Self(self.0 + rhs.0) } diff --git a/crates/brk_vec/src/traits/stored_index.rs b/crates/brk_vec/src/traits/stored_index.rs index c0200f1ed..efb2450dd 100644 --- a/crates/brk_vec/src/traits/stored_index.rs +++ b/crates/brk_vec/src/traits/stored_index.rs @@ -1,5 +1,7 @@ use std::{fmt::Debug, ops::Add}; +use crate::{Error, Result}; + pub trait StoredIndex where Self: Debug @@ -16,8 +18,10 @@ where + Send + Sync, { + fn to_usize(self) -> Result; } -impl StoredIndex for I where +impl StoredIndex for I +where I: Debug + Default + Copy @@ -30,6 +34,10 @@ impl StoredIndex for I where + From + Add + Send - + Sync + + Sync, { + #[inline(always)] + fn to_usize(self) -> Result { + self.try_into().map_err(|_| Error::FailedKeyTryIntoUsize) + } }