Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: part 1

Browse Source
This commit is contained in:
nym21
2025-03-19 12:01:54 +01:00
parent ad761e388d
commit 29c10f8854
39 changed files with 1567 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
Cargo.lock generated
View File

@@ -374,7 +374,7 @@ dependencies = [
[[package]]
name = "brk"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"brk_cli",
"brk_computer",
@@ -391,7 +391,7 @@ dependencies = [
[[package]]
name = "brk_cli"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"brk_computer",
"brk_core",
@@ -412,7 +412,7 @@ dependencies = [
[[package]]
name = "brk_computer"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"brk_core",
"brk_exit",
@@ -427,7 +427,7 @@ dependencies = [
[[package]]
name = "brk_core"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
@@ -444,7 +444,7 @@ dependencies = [
[[package]]
name = "brk_exit"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"brk_logger",
"ctrlc",
@@ -453,7 +453,7 @@ dependencies = [
[[package]]
name = "brk_fetcher"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"brk_core",
"brk_logger",
@@ -466,7 +466,7 @@ dependencies = [
[[package]]
name = "brk_indexer"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
@@ -485,7 +485,7 @@ dependencies = [
[[package]]
name = "brk_logger"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"color-eyre",
"env_logger",
@@ -495,7 +495,7 @@ dependencies = [
[[package]]
name = "brk_parser"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
@@ -510,7 +510,7 @@ dependencies = [
[[package]]
name = "brk_query"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"brk_computer",
"brk_indexer",
@@ -526,7 +526,7 @@ dependencies = [
[[package]]
name = "brk_server"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"axum",
"brk_computer",
@@ -551,7 +551,7 @@ dependencies = [
[[package]]
name = "brk_vec"
version = "0.0.9"
version = "0.0.10"
dependencies = [
"memmap2",
"rayon",
@@ -764,9 +764,9 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "compact_str"
version = "0.8.1"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32"
checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"
dependencies = [
"castaway",
"cfg-if",
@@ -976,9 +976,9 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
[[package]]
name = "deranged"
version = "0.3.11"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"
dependencies = [
"powerfmt",
"serde",
@@ -1217,14 +1217,16 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.3.1"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi 0.13.3+wasi-0.2.2",
"windows-targets",
"r-efi",
"wasi 0.14.2+wasi-0.2.4",
"wasm-bindgen",
]
[[package]]
@@ -1625,6 +1627,17 @@ dependencies = [
"crc",
]
[[package]]
name = "lzma-sys"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "matchit"
version = "0.8.4"
@@ -1782,9 +1795,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
[[package]]
name = "oxc"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45dac9dff4aa3da5b483ec7f7180b0af4a82882c3b35e67c8f9221e117bf0c93"
checksum = "3ceb3e662702022eb9714815de93b138d63e165c6773c1fcaf39e086ef9932ed"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1800,23 +1813,23 @@ dependencies = [
[[package]]
name = "oxc-miette"
version = "1.0.2"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e03e63fd113c068b82d07c9c614b0b146c08a3ac0a4dface3ea1d1a9d14d549e"
checksum = "112f8458565c773a1f6555c14d6cd0255f56c85dd372932029e4baeb15308bbe"
dependencies = [
"cfg-if",
"owo-colors 4.2.0",
"oxc-miette-derive",
"textwrap",
"thiserror 1.0.69",
"thiserror",
"unicode-width",
]
[[package]]
name = "oxc-miette-derive"
version = "1.0.2"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e21f680e8c5f1900297d394627d495351b9e37761f7bbf90116bd5eeb6e80967"
checksum = "4c0c893f53900e3fe01eca3d6d3b54085573c3e48fe25af9d57dd94ef600dcd3"
dependencies = [
"proc-macro2",
"quote",
@@ -1825,9 +1838,9 @@ dependencies = [
[[package]]
name = "oxc_allocator"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9e49310ddfd3bc659d60b9f72bb0fbdb7b23f9bca5b4906056bf1d7d1a502d2"
checksum = "0fb8b78d9f70b5123bb01212458971da4cf0f796ad131602864df51d779290df"
dependencies = [
"allocator-api2",
"assert-unchecked",
@@ -1839,9 +1852,9 @@ dependencies = [
[[package]]
name = "oxc_ast"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54af74d151e1a61d57ec8699f1e8b6729d3817fe763c2ecbacb945822998ea3e"
checksum = "d5cae7318a452457a303b688a1d12735e72e122114ac76bffa029ee34a5d0292"
dependencies = [
"bitflags",
"cow-utils",
@@ -1856,9 +1869,9 @@ dependencies = [
[[package]]
name = "oxc_ast_macros"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d85874efff8c6b1f8b3adf8f3d8624e52ffab8a44da1e2e792de6a0303a9abb8"
checksum = "9ef30e196e021dcf6afd96417b9cd0fe223f0e517107aafdac9648022b280c1f"
dependencies = [
"proc-macro2",
"quote",
@@ -1867,9 +1880,9 @@ dependencies = [
[[package]]
name = "oxc_ast_visit"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03bdf81b8db7952a841d15141e9efc40c8dd01720b9f1779b37f6d3ae5c9e7e4"
checksum = "1ac568f9ce9b9abf31b5f53d6c2f50b3bdb638c71539d177fb566a646cd32cf8"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1879,9 +1892,9 @@ dependencies = [
[[package]]
name = "oxc_cfg"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc06be42ba66c3ab03fb82d973038ee8f5806cf8a7cd23beaa05f262cd63eee3"
checksum = "70c2580450c194b945007dbf2ed99ce167a9c61b0c6039d339c65735c35660fc"
dependencies = [
"bitflags",
"itertools",
@@ -1894,9 +1907,9 @@ dependencies = [
[[package]]
name = "oxc_codegen"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd09d5789bd90a760aa1e1a634fdd20e2b43981b590639264703e6760d33e5e6"
checksum = "cdb54f7e72e0e345e06a79a5c7807d365ab8b8b438742878a0ab8d3bdbba6a0e"
dependencies = [
"bitflags",
"cow-utils",
@@ -1915,19 +1928,18 @@ dependencies = [
[[package]]
name = "oxc_data_structures"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f92e1c4325cef51dda4296fd92302a6c3117325609efb81ac3f0996e7e44977b"
checksum = "1c5ff1571a7cbbaa30abd7dd624274b2fa67bf896d3121e8a0fff2ae50fa889a"
dependencies = [
"assert-unchecked",
"ropey",
]
[[package]]
name = "oxc_diagnostics"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4866163037145687f7197fb70bad1fd0c109e9e2f70659f3eb7f038cd3168bd6"
checksum = "8c44fd92c6601ec6c5d02b210a8196027a0e49eebcc5ea7bcef2efe3c697ad64"
dependencies = [
"cow-utils",
"oxc-miette",
@@ -1935,9 +1947,9 @@ dependencies = [
[[package]]
name = "oxc_ecmascript"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0fbfb5f543a10fb1264a5c24731ab700b5e6f7bedcc2c39792267039824216c"
checksum = "008ecf89e067b5ab8f5e1ac5890059f7c3cd5c9a8ae2db8ef4560dc93f53f01c"
dependencies = [
"cow-utils",
"num-bigint",
@@ -1949,9 +1961,9 @@ dependencies = [
[[package]]
name = "oxc_estree"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84fade441037d1c4f5929f278f7ed74bc5b5928f35f5bc27ffb569512762622d"
checksum = "0e3ec988abe93acf6e99a5ddc3d08116977426fe64c78ed4d5d55852cdf13e14"
[[package]]
name = "oxc_index"
@@ -1961,9 +1973,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
[[package]]
name = "oxc_mangler"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfbf4a5f8bb51ec1b97100b4faaf50379f97d635bba0a39d95aa613c51fe630"
checksum = "be2f21973b9276723af3355830fb620922fa68963bb6617d0733555e98c16ba9"
dependencies = [
"fixedbitset",
"itertools",
@@ -1978,9 +1990,9 @@ dependencies = [
[[package]]
name = "oxc_minifier"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea5c83bfaef4a1d5d8ffdf2b10bceadd57c8b15ccade74ae7a6672c6af31d095"
checksum = "4b6d676deac5a77f7fddadf01e8c1dff89bbe707c8e0f4ec1a734e8227458b5a"
dependencies = [
"cow-utils",
"oxc_allocator",
@@ -2000,9 +2012,9 @@ dependencies = [
[[package]]
name = "oxc_parser"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b702c0462e5a67c845d7fafe236d8bb253ae0f4c8ff44a980b916a154862534"
checksum = "b70aae5a851f227bde01ffe9d6ad7b00cf1b836f62c9ea089da8372f4a30c78c"
dependencies = [
"assert-unchecked",
"bitflags",
@@ -2023,9 +2035,9 @@ dependencies = [
[[package]]
name = "oxc_regular_expression"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b672d8601f80b9828342e0a38f6cc6735a04e2e4251c0c46476df55842df711a"
checksum = "7ed30ec210725f78e8f8c8d1cf597af11b40f18a62d4be5b97d3366ef0baee01"
dependencies = [
"oxc_allocator",
"oxc_ast_macros",
@@ -2039,9 +2051,9 @@ dependencies = [
[[package]]
name = "oxc_semantic"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae29a491046d24b7cbac9ce0602e1c30c52f7065013e7212d716f283800a92ab"
checksum = "8a593327201c47df95293d5c2a85b13debd093b67c5a2ca4b5cbd6628a8e62dd"
dependencies = [
"assert-unchecked",
"itertools",
@@ -2076,9 +2088,9 @@ dependencies = [
[[package]]
name = "oxc_span"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c059e07f57c3299f54dfed3ba2f58dcc183ad68102d1186f8a4c5f546a2b9c5e"
checksum = "d1d4588fc61610c944557dc5b40a93ddbf497a98717f660f2258253ead7978a9"
dependencies = [
"compact_str",
"oxc-miette",
@@ -2089,9 +2101,9 @@ dependencies = [
[[package]]
name = "oxc_syntax"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2888043d4a47ee54903a229f3cfbab1126223c9b819505e900485993467a04d1"
checksum = "c8584c190328b5841abeeeba41342f4a187979eb75098fd506e522927c725211"
dependencies = [
"assert-unchecked",
"bitflags",
@@ -2110,9 +2122,9 @@ dependencies = [
[[package]]
name = "oxc_traverse"
version = "0.58.1"
version = "0.60.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b09630f1f467e91901e1451d651cb8fef0cab7b3facedecc022d6065842cd8a"
checksum = "53fa9743c952a78f5e66050a85f4dd60a549e305fb2aca565442186e7ac8db3c"
dependencies = [
"compact_str",
"itoa",
@@ -2345,6 +2357,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
[[package]]
name = "rand"
version = "0.8.5"
@@ -2462,16 +2480,6 @@ dependencies = [
"libc",
]
[[package]]
name = "ropey"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93411e420bcd1a75ddd1dc3caf18c23155eda2c090631a85af21ba19e97093b5"
dependencies = [
"smallvec",
"str_indices",
]
[[package]]
name = "rustc-demangle"
version = "0.1.24"
@@ -2775,12 +2783,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
[[package]]
name = "str_indices"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6"
[[package]]
name = "strsim"
version = "0.11.1"
@@ -2851,7 +2853,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600"
dependencies = [
"fastrand",
"getrandom 0.3.1",
"getrandom 0.3.2",
"once_cell",
"rustix",
"windows-sys 0.59.0",
@@ -2868,33 +2870,13 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl 1.0.69",
]
[[package]]
name = "thiserror"
version = "2.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
dependencies = [
"thiserror-impl 2.0.12",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"thiserror-impl",
]
[[package]]
@@ -2920,9 +2902,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.39"
version = "0.3.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8"
checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618"
dependencies = [
"deranged",
"itoa",
@@ -2935,15 +2917,15 @@ dependencies = [
[[package]]
name = "time-core"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef"
checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
[[package]]
name = "time-macros"
version = "0.2.20"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c"
checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04"
dependencies = [
"num-conv",
"time-core",
@@ -3205,9 +3187,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasi"
version = "0.13.3+wasi-0.2.2"
version = "0.14.2+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
dependencies = [
"wit-bindgen-rt",
]
@@ -3384,9 +3366,9 @@ dependencies = [
[[package]]
name = "wit-bindgen-rt"
version = "0.33.0"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags",
]
@@ -3397,6 +3379,15 @@ version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
[[package]]
name = "xz2"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
dependencies = [
"lzma-sys",
]
[[package]]
name = "zerocopy"
version = "0.8.23"
@@ -3439,9 +3430,9 @@ dependencies = [
[[package]]
name = "zip"
version = "2.2.3"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b280484c454e74e5fff658bbf7df8fdbe7a07c6b2de4a53def232c15ef138f3a"
checksum = "938cc23ac49778ac8340e366ddc422b2227ea176edb447e23fc0627608dddadd"
dependencies = [
"aes",
"arbitrary",
@@ -3452,15 +3443,16 @@ dependencies = [
"deflate64",
"displaydoc",
"flate2",
"getrandom 0.3.2",
"hmac",
"indexmap 2.8.0",
"lzma-rs",
"memchr",
"pbkdf2",
"rand",
"sha1",
"thiserror 2.0.12",
"thiserror",
"time",
"xz2",
"zeroize",
"zopfli",
"zstd",

2
Cargo.toml
View File

@@ -4,7 +4,7 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.0.9"
package.version = "0.0.10"
package.repository = "https://github.com/bitcoinresearchkit/brk"
[workspace.dependencies]

5
crates/brk_computer/examples/main.rs
View File

@@ -5,10 +5,7 @@ use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{
Parser,
rpc::{self},
};
use brk_parser::{Parser, rpc};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

108
crates/brk_computer/src/storage/vecs/base.rs
View File

@@ -3,13 +3,13 @@ use std::{
cmp::Ordering,
fmt::Debug,
io,
ops::{Add, Deref, DerefMut, Sub},
ops::{Add, Sub},
path::{Path, PathBuf},
};
use brk_core::CheckedSub;
use brk_exit::Exit;
use brk_vec::{Compressed, Error, Result, StoredIndex, StoredType, Version};
use brk_vec::{AnyStorableVec, Compressed, Error, Result, StoredIndex, StoredType, Version};
const FLUSH_EVERY: usize = 10_000;
@@ -24,7 +24,11 @@ where
I: StoredIndex,
T: StoredType,
{
pub fn import(path: &Path, version: Version, compressed: Compressed) -> brk_vec::Result<Self> {
pub fn forced_import(
path: &Path,
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
let vec = brk_vec::StorableVec::forced_import(path, version, compressed)?;
Ok(Self {
@@ -33,24 +37,19 @@ where
})
}
#[inline]
pub fn i_to_usize(index: I) -> Result<usize> {
index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)
}
fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> {
if exit.triggered() {
return Ok(());
}
exit.block();
self.truncate_if_needed(index)?;
self.vec.truncate_if_needed(index)?;
exit.release();
Ok(())
}
#[inline]
fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
match self.len().cmp(&Self::i_to_usize(index)?) {
pub fn forced_push_at(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
match self.len().cmp(&index.to_usize()?) {
Ordering::Less => {
return Err(Error::IndexTooHigh);
}
@@ -58,11 +57,11 @@ where
if ord == Ordering::Greater {
self.safe_truncate_if_needed(index, exit)?;
}
self.push(value);
self.vec.push(value);
}
}
if self.pushed_len() >= FLUSH_EVERY {
if self.vec.pushed_len() >= FLUSH_EVERY {
Ok(self.safe_flush(exit)?)
} else {
Ok(())
@@ -74,20 +73,48 @@ where
return Ok(());
}
exit.block();
self.flush()?;
self.vec.flush()?;
exit.release();
Ok(())
}
fn version(&self) -> Version {
self.vec.version()
}
fn len(&self) -> usize {
self.vec.len()
}
pub fn vec(&self) -> &brk_vec::StorableVec<I, T> {
&self.vec
}
pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec<I, T> {
&mut self.vec
}
pub fn any_vec(&self) -> &dyn AnyStorableVec {
&self.vec
}
pub fn get(&mut self, index: I) -> Result<Option<&T>> {
self.vec.get(index)
}
pub fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
self.vec.collect_range(from, to)
}
#[inline]
fn path_computed_version(&self) -> PathBuf {
self.path().join("computed_version")
self.vec.path().join("computed_version")
}
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
let path = self.path_computed_version();
if version.validate(path.as_ref()).is_err() {
self.reset()?;
self.vec.reset()?;
}
version.write(path.as_ref())?;
Ok(())
@@ -112,7 +139,7 @@ where
let index = max_from.min(A::from(self.len()));
other.iter_from_cloned(index, |(a, b, other)| {
let (i, v) = t((a, b, self, other));
self.push_and_flush_if_needed(i, v, exit)
self.forced_push_at(i, v, exit)
})?;
Ok(self.safe_flush(exit)?)
@@ -132,11 +159,11 @@ where
Version::from(0) + self.version() + other.version(),
)?;
let index = max_from.min(self.read_last()?.cloned().unwrap_or_default());
let index = max_from.min(self.vec.get_last()?.cloned().unwrap_or_default());
other.iter_from(index, |(v, i, ..)| {
let i = *i;
if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) {
self.push_and_flush_if_needed(i, v, exit)
if self.get(i).unwrap().is_none_or(|old_v| *old_v > v) {
self.forced_push_at(i, v, exit)
} else {
Ok(())
}
@@ -162,10 +189,10 @@ where
let index = max_from.min(T::from(self.len()));
first_indexes.iter_from(index, |(value, first_index, ..)| {
let first_index = Self::i_to_usize(*first_index)?;
let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?;
let first_index = (first_index).to_usize()?;
let last_index = (last_indexes.get(value)?.unwrap()).to_usize()?;
(first_index..last_index)
.try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit))
.try_for_each(|index| self.forced_push_at(I::from(index), value, exit))
})?;
Ok(self.safe_flush(exit)?)
@@ -190,13 +217,13 @@ where
let mut prev_index: Option<I> = None;
first_indexes.iter_from(index, |(i, v, ..)| {
if let Some(prev_index) = prev_index.take() {
self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?;
self.forced_push_at(prev_index, v.checked_sub(one).unwrap(), exit)?;
}
prev_index.replace(i);
Ok(())
})?;
if let Some(prev_index) = prev_index {
self.push_and_flush_if_needed(
self.forced_push_at(
prev_index,
T::from(final_len).checked_sub(one).unwrap(),
exit,
@@ -215,7 +242,7 @@ where
) -> Result<()>
where
T: From<T2>,
T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
T2: StoredType + Copy + Add<usize, Output = T2> + CheckedSub<T2> + TryInto<T> + Default,
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
self.validate_computed_version_or_reset_file(
@@ -224,9 +251,11 @@ where
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = last_indexes.read(i)?.unwrap();
let count = *last_index + 1_usize - *first_index;
self.push_and_flush_if_needed(i, count.into(), exit)
let last_index = last_indexes.get(i)?.unwrap();
let count = (*last_index + 1_usize)
.checked_sub(*first_index)
.unwrap_or_default();
self.forced_push_at(i, count.into(), exit)
})?;
Ok(self.safe_flush(exit)?)
@@ -250,11 +279,7 @@ where
let index = max_from.min(I::from(self.len()));
self_to_other.iter_from(index, |(i, other, ..)| {
self.push_and_flush_if_needed(
i,
T::from(other_to_self.read(*other)?.unwrap() == &i),
exit,
)
self.forced_push_at(i, T::from(other_to_self.get(*other)?.unwrap() == &i), exit)
})?;
Ok(self.safe_flush(exit)?)
@@ -278,26 +303,15 @@ where
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(index, first_index, ..)| {
let last_index = last_indexes.read(index)?.unwrap();
let last_index = last_indexes.get(index)?.unwrap();
let count = *last_index + 1_usize - *first_index;
self.push_and_flush_if_needed(index, count.into(), exit)
self.forced_push_at(index, count.into(), exit)
})?;
Ok(self.safe_flush(exit)?)
}
}
impl<I, T> Deref for StorableVec<I, T> {
type Target = brk_vec::StorableVec<I, T>;
fn deref(&self) -> &Self::Target {
&self.vec
}
}
impl<I, T> DerefMut for StorableVec<I, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.vec
}
}
impl<I, T> Clone for StorableVec<I, T>
where
I: StoredIndex,

99
crates/brk_computer/src/storage/vecs/blocks.rs Normal file
View File

@@ -0,0 +1,99 @@
use std::{fs, path::Path};
use brk_core::{CheckedSub, Height, Timestamp};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Compressed, Version};
use super::{
Indexes, StorableVec, indexes,
stats::{StorableVecGeneatorByIndex, StorableVecGeneatorOptions},
};
#[derive(Clone)]
pub struct Vecs {
pub height_to_block_interval: StorableVec<Height, Timestamp>,
pub indexes_to_block_interval_stats: StorableVecGeneatorByIndex<Timestamp>,
pub dateindex_to_block_count: StorableVec<Height, u16>,
pub dateindex_to_total_block_count: StorableVec<Height, u32>,
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
height_to_block_interval: StorableVec::forced_import(
&path.join("height_to_block_interval"),
Version::from(1),
compressed,
)?,
indexes_to_block_interval_stats: StorableVecGeneatorByIndex::forced_import(
&path.join("block_interval"),
compressed,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_minmax()
.add_average(),
)?,
dateindex_to_block_count: StorableVec::forced_import(
&path.join("dateindex_to_block_count"),
Version::from(1),
compressed,
)?,
dateindex_to_total_block_count: StorableVec::forced_import(
&path.join("dateindex_to_total_block_count"),
Version::from(1),
compressed,
)?,
})
}
pub fn compute(
&mut self,
indexer: &mut Indexer,
indexes: &mut indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
let indexer_vecs = indexer.mut_vecs();
self.height_to_block_interval.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.mut_vec(),
|(height, timestamp, height_to_timestamp, ..)| {
let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| {
let prev_timestamp = *height_to_timestamp.get(prev_h).unwrap().unwrap();
if prev_timestamp >= timestamp {
Timestamp::ZERO
} else {
timestamp.checked_sub(prev_timestamp).unwrap()
}
});
(height, interval)
},
exit,
)?;
self.indexes_to_block_interval_stats.compute(
&mut self.height_to_block_interval,
indexer,
indexes,
starting_indexes,
);
Ok(())
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
[
vec![
self.height_to_block_interval.any_vec(),
self.dateindex_to_block_count.any_vec(),
self.dateindex_to_total_block_count.any_vec(),
],
self.indexes_to_block_interval_stats.as_any_vecs(),
]
.concat()
}
}

74
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, ops::Deref, path::Path};
use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Compressed, Value, Version};
use brk_vec::{AnyStorableVec, Compressed, Version};
use super::StorableVec;
@@ -25,62 +25,62 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
dateindex_to_date: StorableVec::import(
dateindex_to_date: StorableVec::forced_import(
&path.join("dateindex_to_date"),
Version::from(1),
compressed,
)?,
dateindex_to_dateindex: StorableVec::import(
dateindex_to_dateindex: StorableVec::forced_import(
&path.join("dateindex_to_dateindex"),
Version::from(1),
compressed,
)?,
dateindex_to_first_height: StorableVec::import(
dateindex_to_first_height: StorableVec::forced_import(
&path.join("dateindex_to_first_height"),
Version::from(1),
compressed,
)?,
dateindex_to_last_height: StorableVec::import(
dateindex_to_last_height: StorableVec::forced_import(
&path.join("dateindex_to_last_height"),
Version::from(1),
compressed,
)?,
height_to_real_date: StorableVec::import(
height_to_real_date: StorableVec::forced_import(
&path.join("height_to_real_date"),
Version::from(1),
compressed,
)?,
height_to_fixed_date: StorableVec::import(
height_to_fixed_date: StorableVec::forced_import(
&path.join("height_to_fixed_date"),
Version::from(1),
compressed,
)?,
height_to_dateindex: StorableVec::import(
height_to_dateindex: StorableVec::forced_import(
&path.join("height_to_dateindex"),
Version::from(1),
compressed,
)?,
height_to_height: StorableVec::import(
height_to_height: StorableVec::forced_import(
&path.join("height_to_height"),
Version::from(1),
compressed,
)?,
height_to_last_txindex: StorableVec::import(
height_to_last_txindex: StorableVec::forced_import(
&path.join("height_to_last_txindex"),
Version::from(1),
compressed,
)?,
txindex_to_last_txinindex: StorableVec::import(
txindex_to_last_txinindex: StorableVec::forced_import(
&path.join("txindex_to_last_txinindex"),
Version::from(1),
compressed,
)?,
txindex_to_last_txoutindex: StorableVec::import(
txindex_to_last_txoutindex: StorableVec::forced_import(
&path.join("txindex_to_last_txoutindex"),
Version::from(1),
compressed,
@@ -103,25 +103,25 @@ impl Vecs {
self.height_to_height.compute_transform(
starting_indexes.height,
&mut indexer_vecs.height_to_timestamp,
indexer_vecs.height_to_timestamp.mut_vec(),
|(h, ..)| (h, h),
exit,
)?;
self.height_to_real_date.compute_transform(
starting_indexes.height,
&mut indexer_vecs.height_to_timestamp,
indexer_vecs.height_to_timestamp.mut_vec(),
|(h, t, ..)| (h, Date::from(t)),
exit,
)?;
self.height_to_fixed_date.compute_transform(
starting_indexes.height,
&mut self.height_to_real_date,
self.height_to_real_date.mut_vec(),
|(h, d, s, ..)| {
let d = h
.decremented()
.and_then(|h| s.read(h).ok())
.and_then(|h| s.get(h).ok())
.flatten()
.map_or(d, |prev_d| {
let prev_d = *prev_d;
@@ -134,7 +134,7 @@ impl Vecs {
self.height_to_dateindex.compute_transform(
starting_indexes.height,
&mut self.height_to_fixed_date,
self.height_to_fixed_date.mut_vec(),
|(h, d, ..)| (h, Dateindex::try_from(d).unwrap()),
exit,
)?;
@@ -142,34 +142,34 @@ impl Vecs {
let starting_dateindex = self
.height_to_dateindex
.get(starting_indexes.height.decremented().unwrap_or_default())?
.map(Value::into_inner)
.copied()
.unwrap_or_default();
self.dateindex_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
&mut self.height_to_dateindex,
self.height_to_dateindex.mut_vec(),
exit,
)?;
self.dateindex_to_last_height
.compute_last_index_from_first(
starting_dateindex,
&mut self.dateindex_to_first_height,
self.dateindex_to_first_height.mut_vec(),
height_count,
exit,
)?;
self.dateindex_to_dateindex.compute_transform(
starting_dateindex,
&mut self.dateindex_to_first_height,
self.dateindex_to_first_height.mut_vec(),
|(di, ..)| (di, di),
exit,
)?;
self.dateindex_to_date.compute_transform(
starting_dateindex,
&mut self.dateindex_to_dateindex,
self.dateindex_to_dateindex.mut_vec(),
|(di, ..)| (di, Date::from(di)),
exit,
)?;
@@ -177,7 +177,7 @@ impl Vecs {
self.txindex_to_last_txinindex
.compute_last_index_from_first(
starting_indexes.txindex,
&mut indexer_vecs.txindex_to_first_txinindex,
indexer_vecs.txindex_to_first_txinindex.mut_vec(),
txinindexes_count,
exit,
)?;
@@ -185,14 +185,14 @@ impl Vecs {
self.txindex_to_last_txoutindex
.compute_last_index_from_first(
starting_indexes.txindex,
&mut indexer_vecs.txindex_to_first_txoutindex,
indexer_vecs.txindex_to_first_txoutindex.mut_vec(),
txoutindexes_count,
exit,
)?;
self.height_to_last_txindex.compute_last_index_from_first(
starting_indexes.height,
&mut indexer_vecs.height_to_first_txindex,
indexer_vecs.height_to_first_txindex.mut_vec(),
txindexes_count,
exit,
)?;
@@ -202,17 +202,17 @@ impl Vecs {
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
vec![
&*self.dateindex_to_date,
&*self.dateindex_to_dateindex,
&*self.dateindex_to_first_height,
&*self.dateindex_to_last_height,
&*self.height_to_dateindex,
&*self.height_to_fixed_date,
&*self.height_to_height,
&*self.height_to_last_txindex,
&*self.height_to_real_date,
&*self.txindex_to_last_txinindex,
&*self.txindex_to_last_txoutindex,
self.dateindex_to_date.any_vec(),
self.dateindex_to_dateindex.any_vec(),
self.dateindex_to_first_height.any_vec(),
self.dateindex_to_last_height.any_vec(),
self.height_to_dateindex.any_vec(),
self.height_to_fixed_date.any_vec(),
self.height_to_height.any_vec(),
self.height_to_last_txindex.any_vec(),
self.height_to_real_date.any_vec(),
self.txindex_to_last_txinindex.any_vec(),
self.txindex_to_last_txoutindex.any_vec(),
]
}
}

145
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -6,7 +6,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Compressed, Value, Version};
use brk_vec::{AnyStorableVec, Compressed, Version};
use super::{Indexes, StorableVec, indexes};
@@ -37,116 +37,116 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
dateindex_to_ohlc_in_cents: StorableVec::import(
dateindex_to_ohlc_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_ohlc_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_ohlc: StorableVec::import(
dateindex_to_ohlc: StorableVec::forced_import(
&path.join("dateindex_to_ohlc"),
Version::from(1),
compressed,
)?,
dateindex_to_close_in_cents: StorableVec::import(
dateindex_to_close_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_close_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_close: StorableVec::import(
dateindex_to_close: StorableVec::forced_import(
&path.join("dateindex_to_close"),
Version::from(1),
compressed,
)?,
dateindex_to_high_in_cents: StorableVec::import(
dateindex_to_high_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_high_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_high: StorableVec::import(
dateindex_to_high: StorableVec::forced_import(
&path.join("dateindex_to_high"),
Version::from(1),
compressed,
)?,
dateindex_to_low_in_cents: StorableVec::import(
dateindex_to_low_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_low_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_low: StorableVec::import(
dateindex_to_low: StorableVec::forced_import(
&path.join("dateindex_to_low"),
Version::from(1),
compressed,
)?,
dateindex_to_open_in_cents: StorableVec::import(
dateindex_to_open_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_open_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_open: StorableVec::import(
dateindex_to_open: StorableVec::forced_import(
&path.join("dateindex_to_open"),
Version::from(1),
compressed,
)?,
dateindex_to_sats_per_dollar: StorableVec::import(
dateindex_to_sats_per_dollar: StorableVec::forced_import(
&path.join("dateindex_to_sats_per_dollar"),
Version::from(1),
compressed,
)?,
height_to_ohlc_in_cents: StorableVec::import(
height_to_ohlc_in_cents: StorableVec::forced_import(
&path.join("height_to_ohlc_in_cents"),
Version::from(1),
compressed,
)?,
height_to_ohlc: StorableVec::import(
height_to_ohlc: StorableVec::forced_import(
&path.join("height_to_ohlc"),
Version::from(1),
compressed,
)?,
height_to_close_in_cents: StorableVec::import(
height_to_close_in_cents: StorableVec::forced_import(
&path.join("height_to_close_in_cents"),
Version::from(1),
compressed,
)?,
height_to_close: StorableVec::import(
height_to_close: StorableVec::forced_import(
&path.join("height_to_close"),
Version::from(1),
compressed,
)?,
height_to_high_in_cents: StorableVec::import(
height_to_high_in_cents: StorableVec::forced_import(
&path.join("height_to_high_in_cents"),
Version::from(1),
compressed,
)?,
height_to_high: StorableVec::import(
height_to_high: StorableVec::forced_import(
&path.join("height_to_high"),
Version::from(1),
compressed,
)?,
height_to_low_in_cents: StorableVec::import(
height_to_low_in_cents: StorableVec::forced_import(
&path.join("height_to_low_in_cents"),
Version::from(1),
compressed,
)?,
height_to_low: StorableVec::import(
height_to_low: StorableVec::forced_import(
&path.join("height_to_low"),
Version::from(1),
compressed,
)?,
height_to_open_in_cents: StorableVec::import(
height_to_open_in_cents: StorableVec::forced_import(
&path.join("height_to_open_in_cents"),
Version::from(1),
compressed,
)?,
height_to_open: StorableVec::import(
height_to_open: StorableVec::forced_import(
&path.join("height_to_open"),
Version::from(1),
compressed,
)?,
height_to_sats_per_dollar: StorableVec::import(
height_to_sats_per_dollar: StorableVec::forced_import(
&path.join("height_to_sats_per_dollar"),
Version::from(1),
compressed,
@@ -166,19 +166,14 @@ impl Vecs {
self.height_to_ohlc_in_cents.compute_transform(
starting_indexes.height,
&mut indexer_vecs.height_to_timestamp,
indexer_vecs.height_to_timestamp.mut_vec(),
|(h, t, _, height_to_timestamp)| {
let ohlc = fetcher
.get_height(
h,
t,
h.decremented().map(|prev_h| {
height_to_timestamp
.get(prev_h)
.unwrap()
.map(Value::into_inner)
.unwrap()
}),
h.decremented()
.map(|prev_h| *height_to_timestamp.get(prev_h).unwrap().unwrap()),
)
.unwrap();
(h, ohlc)
@@ -188,77 +183,77 @@ impl Vecs {
self.height_to_open_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
self.height_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.height_to_high_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
self.height_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.height_to_low_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
self.height_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.height_to_close_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
self.height_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.height_to_ohlc.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
self.height_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
self.height_to_open.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
self.height_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.height_to_high.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
self.height_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.height_to_low.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
self.height_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.height_to_close.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
self.height_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.height_to_sats_per_dollar.compute_transform(
starting_indexes.height,
&mut self.height_to_close,
self.height_to_close.mut_vec(),
|(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)),
exit,
)?;
self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex,
&mut indexes.dateindex_to_date,
indexes.dateindex_to_date.mut_vec(),
|(di, d, ..)| {
let ohlc = fetcher.get_date(d).unwrap();
(di, ohlc)
@@ -268,70 +263,70 @@ impl Vecs {
self.dateindex_to_open_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
self.dateindex_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.dateindex_to_high_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
self.dateindex_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.dateindex_to_low_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
self.dateindex_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.dateindex_to_close_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
self.dateindex_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.dateindex_to_ohlc.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
self.dateindex_to_ohlc_in_cents.mut_vec(),
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
self.dateindex_to_open.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
self.dateindex_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.dateindex_to_high.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
self.dateindex_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.dateindex_to_low.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
self.dateindex_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.dateindex_to_close.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
self.dateindex_to_ohlc.mut_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.dateindex_to_sats_per_dollar.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_close,
self.dateindex_to_close.mut_vec(),
|(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)),
exit,
)?;
@@ -341,28 +336,28 @@ impl Vecs {
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
vec![
&*self.dateindex_to_close,
&*self.dateindex_to_close_in_cents,
&*self.dateindex_to_high,
&*self.dateindex_to_high_in_cents,
&*self.dateindex_to_low,
&*self.dateindex_to_low_in_cents,
&*self.dateindex_to_ohlc,
&*self.dateindex_to_ohlc_in_cents,
&*self.dateindex_to_open,
&*self.dateindex_to_open_in_cents,
&*self.dateindex_to_sats_per_dollar,
&*self.height_to_close,
&*self.height_to_close_in_cents,
&*self.height_to_high,
&*self.height_to_high_in_cents,
&*self.height_to_low,
&*self.height_to_low_in_cents,
&*self.height_to_ohlc,
&*self.height_to_ohlc_in_cents,
&*self.height_to_open,
&*self.height_to_open_in_cents,
&*self.height_to_sats_per_dollar,
self.dateindex_to_close.any_vec(),
self.dateindex_to_close_in_cents.any_vec(),
self.dateindex_to_high.any_vec(),
self.dateindex_to_high_in_cents.any_vec(),
self.dateindex_to_low.any_vec(),
self.dateindex_to_low_in_cents.any_vec(),
self.dateindex_to_ohlc.any_vec(),
self.dateindex_to_ohlc_in_cents.any_vec(),
self.dateindex_to_open.any_vec(),
self.dateindex_to_open_in_cents.any_vec(),
self.dateindex_to_sats_per_dollar.any_vec(),
self.height_to_close.any_vec(),
self.height_to_close_in_cents.any_vec(),
self.height_to_high.any_vec(),
self.height_to_high_in_cents.any_vec(),
self.height_to_low.any_vec(),
self.height_to_low_in_cents.any_vec(),
self.height_to_ohlc.any_vec(),
self.height_to_ohlc_in_cents.any_vec(),
self.height_to_open.any_vec(),
self.height_to_open_in_cents.any_vec(),
self.height_to_sats_per_dollar.any_vec(),
]
}
}

13
crates/brk_computer/src/storage/vecs/mod.rs
View File

@@ -6,8 +6,10 @@ use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Compressed};
mod base;
mod blocks;
mod indexes;
mod marketprice;
mod stats;
mod transactions;
use base::*;
@@ -15,6 +17,7 @@ use indexes::*;
#[derive(Clone)]
pub struct Vecs {
pub blocks: blocks::Vecs,
pub indexes: indexes::Vecs,
pub transactions: transactions::Vecs,
pub marketprice: Option<marketprice::Vecs>,
@@ -25,9 +28,10 @@ impl Vecs {
fs::create_dir_all(path)?;
Ok(Self {
indexes: indexes::Vecs::import(path, compressed)?,
transactions: transactions::Vecs::import(path, compressed)?,
marketprice: fetch.then(|| marketprice::Vecs::import(path, compressed).unwrap()),
blocks: blocks::Vecs::forced_import(path, compressed)?,
indexes: indexes::Vecs::forced_import(path, compressed)?,
transactions: transactions::Vecs::forced_import(path, compressed)?,
marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()),
})
}
@@ -40,6 +44,9 @@ impl Vecs {
) -> color_eyre::Result<()> {
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
self.blocks
.compute(indexer, &mut self.indexes, &starting_indexes, exit)?;
self.transactions
.compute(indexer, &mut self.indexes, &starting_indexes, exit)?;

396
crates/brk_computer/src/storage/vecs/stats/generic.rs Normal file
View File

@@ -0,0 +1,396 @@
use std::path::Path;
use brk_exit::Exit;
use brk_vec::{AnyStorableVec, Compressed, Result, StoredIndex, Version};
use crate::storage::vecs::base::StorableVec;
use super::StoredType;
#[derive(Clone, Debug)]
pub struct StorableVecGeneator<I, T>
where
I: StoredIndex,
T: StoredType,
{
// first: Option<StorableVec<I, T>>,
average: Option<StorableVec<I, T>>,
sum: Option<StorableVec<I, T>>,
max: Option<StorableVec<I, T>>,
_90p: Option<StorableVec<I, T>>,
_75p: Option<StorableVec<I, T>>,
median: Option<StorableVec<I, T>>,
_25p: Option<StorableVec<I, T>>,
_10p: Option<StorableVec<I, T>>,
min: Option<StorableVec<I, T>>,
last: Option<StorableVec<I, T>>,
}
impl<I, T> StorableVecGeneator<I, T>
where
I: StoredIndex,
T: StoredType,
{
pub fn forced_import(
path: &Path,
compressed: Compressed,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let name = path.file_name().unwrap().to_str().unwrap().to_string();
// let prefix = |s: &str| path.with_file_name(format!("{s}_{name}"));
let suffix = |s: &str| path.with_file_name(format!("{name}_{s}"));
let s = Self {
// first: options.first.then(|| {
// StorableVec::forced_import(&prefix("first"), Version::from(1), compressed).unwrap()
// }),
last: options
.last
.then(|| StorableVec::forced_import(path, Version::from(1), compressed).unwrap()),
min: options.min.then(|| {
StorableVec::forced_import(&suffix("min"), Version::from(1), compressed).unwrap()
}),
max: options.max.then(|| {
StorableVec::forced_import(&suffix("max"), Version::from(1), compressed).unwrap()
}),
median: options.median.then(|| {
StorableVec::forced_import(&suffix("median"), Version::from(1), compressed).unwrap()
}),
average: options.average.then(|| {
StorableVec::forced_import(&suffix("average"), Version::from(1), compressed)
.unwrap()
}),
sum: options.sum.then(|| {
StorableVec::forced_import(&suffix("sum"), Version::from(1), compressed).unwrap()
}),
_90p: options._90p.then(|| {
StorableVec::forced_import(&suffix("90p"), Version::from(1), compressed).unwrap()
}),
_75p: options._75p.then(|| {
StorableVec::forced_import(&suffix("75p"), Version::from(1), compressed).unwrap()
}),
_25p: options._25p.then(|| {
StorableVec::forced_import(&suffix("25p"), Version::from(1), compressed).unwrap()
}),
_10p: options._10p.then(|| {
StorableVec::forced_import(&suffix("10p"), Version::from(1), compressed).unwrap()
}),
};
Ok(s)
}
pub fn compute<I2>(
&mut self,
max_from: I,
source: &mut StorableVec<I2, T>,
first_indexes: &mut brk_vec::StorableVec<I, I2>,
last_indexes: &mut brk_vec::StorableVec<I, I2>,
exit: &Exit,
) -> Result<()>
where
I2: StoredIndex + StoredType,
T: Ord,
{
let index = self.starting_index(max_from);
first_indexes.iter_from(index, |(i, first_index)| {
let first_index = *first_index;
let last_index = *last_indexes.get(i).unwrap().unwrap();
// if let Some(first) = self.first.as_mut() {
// let v = source.read(first_index).unwrap().unwrap();
// first.forced_push_at(index, v.clone(), exit);
// }
if let Some(last) = self.last.as_mut() {
let v = source.get(last_index).unwrap().unwrap();
last.forced_push_at(index, v.clone(), exit)?;
}
let first_index = first_index.to_usize()?;
let last_index = last_index.to_usize()?;
let needs_sum_or_average = self.sum.is_some() || self.average.is_some();
let needs_sorted = self.max.is_some()
|| self._90p.is_some()
|| self._75p.is_some()
|| self.median.is_some()
|| self._25p.is_some()
|| self._10p.is_some()
|| self.min.is_some();
let needs_values = needs_sorted || needs_sum_or_average;
if needs_values {
let mut values =
source.collect_range(Some(first_index as i64), Some(last_index as i64))?;
if needs_sorted {
values.sort_unstable();
if let Some(max) = self.max.as_mut() {
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.forced_push_at(i, Self::get_percentile(&values, 0.90), exit)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.forced_push_at(i, Self::get_percentile(&values, 0.75), exit)?;
}
if let Some(median) = self.median.as_mut() {
median.forced_push_at(i, Self::get_percentile(&values, 0.50), exit)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.forced_push_at(i, Self::get_percentile(&values, 0.25), exit)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.forced_push_at(i, Self::get_percentile(&values, 0.10), exit)?;
}
if let Some(min) = self.min.as_mut() {
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
}
}
if needs_sum_or_average {
let len = values.len();
if let Some(average) = self.average.as_mut() {
let a = values
.iter()
.map(|v| v.clone() / len)
.fold(T::from(0), |a, b| a + b);
average.forced_push_at(i, a, exit)?;
}
if let Some(sum_vec) = self.sum.as_mut() {
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
sum_vec.forced_push_at(i, sum, exit)?;
}
}
}
Ok(())
})?;
Ok(())
}
fn get_percentile(sorted: &[T], percentile: f64) -> T {
let len = sorted.len();
if len == 0 {
panic!();
} else if len == 1 {
sorted[0].clone()
} else {
let index = (len - 1) as f64 * percentile;
let fract = index.fract();
if fract != 0.0 {
let left = sorted.get(index as usize).unwrap().clone();
let right = sorted.get(index.ceil() as usize).unwrap().clone();
left / 2 + right / 2
} else {
sorted.get(index as usize).unwrap().clone()
}
}
}
fn starting_index(&self, max_from: I) -> I {
max_from.min(I::from(
self.as_any_vecs()
.into_iter()
.map(|v| v.len())
.min()
.unwrap(),
))
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
let mut v: Vec<&dyn AnyStorableVec> = vec![];
// if let Some(first) = self.first.as_ref() {
// v.push(first.any_vec());
// }
if let Some(last) = self.last.as_ref() {
v.push(last.any_vec());
}
if let Some(min) = self.min.as_ref() {
v.push(min.any_vec());
}
if let Some(max) = self.max.as_ref() {
v.push(max.any_vec());
}
if let Some(median) = self.median.as_ref() {
v.push(median.any_vec());
}
if let Some(average) = self.average.as_ref() {
v.push(average.any_vec());
}
if let Some(sum) = self.sum.as_ref() {
v.push(sum.any_vec());
}
if let Some(_90p) = self._90p.as_ref() {
v.push(_90p.any_vec());
}
if let Some(_75p) = self._75p.as_ref() {
v.push(_75p.any_vec());
}
if let Some(_25p) = self._25p.as_ref() {
v.push(_25p.any_vec());
}
if let Some(_10p) = self._10p.as_ref() {
v.push(_10p.any_vec());
}
v
}
}
#[derive(Default, Clone, Copy)]
pub struct StorableVecGeneatorOptions {
average: bool,
sum: bool,
max: bool,
_90p: bool,
_75p: bool,
median: bool,
_25p: bool,
_10p: bool,
min: bool,
// first: bool,
last: bool,
}
impl StorableVecGeneatorOptions {
// pub fn add_first(mut self) -> Self {
// self.first = true;
// self
// }
pub fn add_last(mut self) -> Self {
self.last = true;
self
}
pub fn add_min(mut self) -> Self {
self.min = true;
self
}
pub fn add_max(mut self) -> Self {
self.max = true;
self
}
pub fn add_median(mut self) -> Self {
self.median = true;
self
}
pub fn add_average(mut self) -> Self {
self.average = true;
self
}
pub fn add_sum(mut self) -> Self {
self.sum = true;
self
}
pub fn add_90p(mut self) -> Self {
self._90p = true;
self
}
pub fn add_75p(mut self) -> Self {
self._75p = true;
self
}
pub fn add_25p(mut self) -> Self {
self._25p = true;
self
}
pub fn add_10p(mut self) -> Self {
self._10p = true;
self
}
pub fn rm_min(mut self) -> Self {
self.min = false;
self
}
pub fn rm_max(mut self) -> Self {
self.max = false;
self
}
pub fn rm_median(mut self) -> Self {
self.median = false;
self
}
pub fn rm_average(mut self) -> Self {
self.average = false;
self
}
pub fn rm_sum(mut self) -> Self {
self.sum = false;
self
}
pub fn rm_90p(mut self) -> Self {
self._90p = false;
self
}
pub fn rm_75p(mut self) -> Self {
self._75p = false;
self
}
pub fn rm_25p(mut self) -> Self {
self._25p = false;
self
}
pub fn rm_10p(mut self) -> Self {
self._10p = false;
self
}
pub fn add_minmax(mut self) -> Self {
self.min = true;
self.max = true;
self
}
pub fn add_percentiles(mut self) -> Self {
self._90p = true;
self._75p = true;
self.median = true;
self._25p = true;
self._10p = true;
self
}
pub fn remove_percentiles(mut self) -> Self {
self._90p = false;
self._75p = false;
self.median = false;
self._25p = false;
self._10p = false;
self
}
}

72
crates/brk_computer/src/storage/vecs/stats/grouped.rs Normal file
View File

@@ -0,0 +1,72 @@
use std::path::Path;
use brk_core::{
Dateindex, Decadeindex, Difficultyepoch, Halvingepoch, Height, Monthindex, Weekindex, Yearindex,
};
use brk_indexer::{Indexer, Indexes};
use brk_vec::{AnyStorableVec, Compressed};
use crate::storage::vecs::{base::StorableVec, indexes};
use super::{StorableVecGeneator, StorableVecGeneatorOptions, StoredType};
#[derive(Clone)]
pub struct StorableVecGeneatorByIndex<T>
where
T: StoredType,
{
pub dateindex: StorableVecGeneator<Dateindex, T>,
pub weekindex: StorableVecGeneator<Weekindex, T>,
pub difficultyepoch: StorableVecGeneator<Difficultyepoch, T>,
pub monthindex: StorableVecGeneator<Monthindex, T>,
pub yearindex: StorableVecGeneator<Yearindex, T>,
pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
pub decadeindex: StorableVecGeneator<Decadeindex, T>,
}
impl<T> StorableVecGeneatorByIndex<T>
where
T: StoredType,
{
pub fn forced_import(
path: &Path,
compressed: Compressed,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let dateindex = StorableVecGeneator::forced_import(path, compressed, options)?;
let options = options.remove_percentiles();
Ok(Self {
dateindex,
weekindex: StorableVecGeneator::forced_import(path, compressed, options)?,
difficultyepoch: StorableVecGeneator::forced_import(path, compressed, options)?,
monthindex: StorableVecGeneator::forced_import(path, compressed, options)?,
yearindex: StorableVecGeneator::forced_import(path, compressed, options)?,
halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?,
decadeindex: StorableVecGeneator::forced_import(path, compressed, options)?,
})
}
pub fn compute(
&mut self,
source: &mut StorableVec<Height, T>,
indexer: &mut Indexer,
indexes: &mut indexes::Vecs,
starting_indexes: &Indexes,
) {
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
[
self.dateindex.as_any_vecs(),
self.weekindex.as_any_vecs(),
self.difficultyepoch.as_any_vecs(),
self.monthindex.as_any_vecs(),
self.yearindex.as_any_vecs(),
self.halvingepoch.as_any_vecs(),
self.decadeindex.as_any_vecs(),
]
.concat()
}
}

7
crates/brk_computer/src/storage/vecs/stats/mod.rs Normal file
View File

@@ -0,0 +1,7 @@
mod generic;
mod grouped;
mod stored_type;
pub use generic::*;
pub use grouped::*;
pub use stored_type::*;

11
crates/brk_computer/src/storage/vecs/stats/stored_type.rs Normal file
View File

@@ -0,0 +1,11 @@
use std::ops::{Add, Div};
pub trait StoredType
where
Self: brk_vec::StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self>,
{
}
impl<T> StoredType for T where
T: brk_vec::StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self>
{
}

94
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -9,7 +9,6 @@ use super::{Indexes, StorableVec, indexes};
#[derive(Clone)]
pub struct Vecs {
// pub height_to_block_interval: StorableVec<Height, Timestamp>,
// pub height_to_fee: StorableVec<Txindex, Sats>,
// pub height_to_inputcount: StorableVec<Height, u32>,
// pub height_to_maxfeerate: StorableVec<Height, Feerate>,
@@ -22,24 +21,30 @@ pub struct Vecs {
// pub txindex_to_fee: StorableVec<Txindex, Sats>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
// pub txindex_to_inputs_count: StorableVec<Txindex, u32>,
pub txindex_to_inputs_count: StorableVec<Txindex, u32>,
// pub txindex_to_inputs_sum: StorableVec<Txindex, Sats>,
// pub txindex_to_outputs_count: StorableVec<Txindex, u32>,
pub txindex_to_outputs_count: StorableVec<Txindex, u32>,
// pub txindex_to_outputs_sum: StorableVec<Txindex, Sats>,
// pub txinindex_to_value: StorableVec<Txinindex, Sats>,
}
impl Vecs {
pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
// height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_input_count: StorableVec::forced_import(
// &path.join("height_to_input_count"),
// Version::from(1),
// )?,
// height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?,
// height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?,
// height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?,
// height_to_output_count: StorableVec::forced_import(
// &path.join("height_to_output_count"),
// Version::from(1),
// )?,
// height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?,
// height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?,
@@ -47,28 +52,35 @@ impl Vecs {
// &path.join("txindex_to_fee"),
// Version::from(1),
// )?,
txindex_to_is_coinbase: StorableVec::import(
txindex_to_is_coinbase: StorableVec::forced_import(
&path.join("txindex_to_is_coinbase"),
Version::from(1),
compressed,
)?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?,
// txindex_to_inputs_count: StorableVec::forced_import(
// &path.join("txindex_to_inputs_count"),
// Version::from(1),
// )?,
txindex_to_inputs_count: StorableVec::forced_import(
&path.join("txindex_to_inputs_count"),
Version::from(1),
compressed,
)?,
// txindex_to_inputs_sum: StorableVec::forced_import(
// &path.join("txindex_to_inputs_sum"),
// Version::from(1),
// )?,
// txindex_to_outputs_count: StorableVec::forced_import(
// &path.join("txindex_to_outputs_count"),
// Version::from(1),
// )?,
txindex_to_outputs_count: StorableVec::forced_import(
&path.join("txindex_to_outputs_count"),
Version::from(1),
compressed,
)?,
// txindex_to_outputs_sum: StorableVec::forced_import(
// &path.join("txindex_to_outputs_sum"),
// Version::from(1),
// )?,
// txinindex_to_value: StorableVec::forced_import(
// &path.join("txinindex_to_value"),
// Version::from(1),
// compressed,
// )?,
})
}
@@ -81,27 +93,43 @@ impl Vecs {
) -> color_eyre::Result<()> {
let indexer_vecs = indexer.mut_vecs();
// self.vecs.txindex_to_inputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// &mut self.vecs.txindex_to_last_txinindex,
// exit,
// )?;
self.txindex_to_inputs_count.compute_count_from_indexes(
starting_indexes.txindex,
indexer_vecs.txindex_to_first_txinindex.mut_vec(),
indexes.txindex_to_last_txinindex.mut_vec(),
exit,
)?;
// self.vecs.txindex_to_outputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// &mut self.vecs.txindex_to_last_txoutindex,
// exit,
// )?;
self.txindex_to_outputs_count.compute_count_from_indexes(
starting_indexes.txindex,
indexer_vecs.txindex_to_first_txoutindex.mut_vec(),
indexes.txindex_to_last_txoutindex.mut_vec(),
exit,
)?;
self.txindex_to_is_coinbase.compute_is_first_ordered(
starting_indexes.txindex,
&mut indexer_vecs.txindex_to_height,
&mut indexer_vecs.height_to_first_txindex,
indexer_vecs.txindex_to_height.mut_vec(),
indexer_vecs.height_to_first_txindex.mut_vec(),
exit,
)?;
// self.txinindex_to_value.compute_transform(
// starting_indexes.txinindex,
// indexer_vecs.txinindex_to_txoutindex.mut_vec(),
// |(txinindex, txoutindex, slf, other)| {
// let value =
// if let Ok(Some(value)) = indexer_vecs.txoutindex_to_value.read(txoutindex) {
// *value
// } else {
// dbg!(txinindex, txoutindex, slf.len(), other.len());
// panic!()
// };
// (txinindex, value)
// },
// exit,
// )?;
// self.vecs.txindex_to_fee.compute_transform(
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs().height_to_first_txindex,
@@ -128,6 +156,10 @@ impl Vecs {
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
vec![&*self.txindex_to_is_coinbase]
vec![
self.txindex_to_is_coinbase.any_vec(),
self.txindex_to_inputs_count.any_vec(),
self.txindex_to_outputs_count.any_vec(),
]
}
}

48
crates/brk_core/src/structs/decadeindex.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{fmt::Debug, ops::Add};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
Serialize,
Deserialize,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Decadeindex(u8);
impl From<u8> for Decadeindex {
fn from(value: u8) -> Self {
Self(value)
}
}
impl From<usize> for Decadeindex {
fn from(value: usize) -> Self {
Self(value as u8)
}
}
impl From<Decadeindex> for usize {
fn from(value: Decadeindex) -> Self {
value.0 as usize
}
}
impl Add<usize> for Decadeindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u8)
}
}

48
crates/brk_core/src/structs/difficultyepoch.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{fmt::Debug, ops::Add};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
Serialize,
Deserialize,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Difficultyepoch(u16);
impl From<u16> for Difficultyepoch {
fn from(value: u16) -> Self {
Self(value)
}
}
impl From<usize> for Difficultyepoch {
fn from(value: usize) -> Self {
Self(value as u16)
}
}
impl From<Difficultyepoch> for usize {
fn from(value: Difficultyepoch) -> Self {
value.0 as usize
}
}
impl Add<usize> for Difficultyepoch {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u16)
}
}

48
crates/brk_core/src/structs/halvingepoch.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{fmt::Debug, ops::Add};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
Serialize,
Deserialize,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Halvingepoch(u8);
impl From<u8> for Halvingepoch {
fn from(value: u8) -> Self {
Self(value)
}
}
impl From<usize> for Halvingepoch {
fn from(value: usize) -> Self {
Self(value as u8)
}
}
impl From<Halvingepoch> for usize {
fn from(value: Halvingepoch) -> Self {
value.0 as usize
}
}
impl Add<usize> for Halvingepoch {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u8)
}
}

14
crates/brk_core/src/structs/height.rs
View File

@@ -67,7 +67,7 @@ impl PartialEq<u64> for Height {
}
impl Add<Height> for Height {
type Output = Height;
type Output = Self;
fn add(self, rhs: Height) -> Self::Output {
Self::from(self.0 + rhs.0)
@@ -75,7 +75,7 @@ impl Add<Height> for Height {
}
impl Add<u32> for Height {
type Output = Height;
type Output = Self;
fn add(self, rhs: u32) -> Self::Output {
Self::from(self.0 + rhs)
@@ -83,7 +83,7 @@ impl Add<u32> for Height {
}
impl Add<usize> for Height {
type Output = Height;
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u32)
@@ -91,8 +91,8 @@ impl Add<usize> for Height {
}
impl CheckedSub<Height> for Height {
fn checked_sub(self, rhs: Height) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Height::from)
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self::from)
}
}
@@ -115,14 +115,14 @@ impl AddAssign<usize> for Height {
}
impl Rem<Height> for Height {
type Output = Height;
type Output = Self;
fn rem(self, rhs: Height) -> Self::Output {
Self(self.0.rem(rhs.0))
}
}
impl Rem<usize> for Height {
type Output = Height;
type Output = Self;
fn rem(self, rhs: usize) -> Self::Output {
Self(self.0.rem(Height::from(rhs).0))
}

12
crates/brk_core/src/structs/mod.rs
View File

@@ -9,10 +9,14 @@ mod cents;
mod compressed;
mod date;
mod dateindex;
mod decadeindex;
mod difficultyepoch;
mod dollars;
mod feerate;
mod halvingepoch;
mod height;
mod locktime;
mod monthindex;
mod ohlc;
mod sats;
mod timestamp;
@@ -24,7 +28,9 @@ mod txversion;
mod unit;
mod vin;
mod vout;
mod weekindex;
mod weight;
mod yearindex;
pub use addressbytes::*;
pub use addressindex::*;
@@ -37,10 +43,14 @@ pub use cents::*;
pub use compressed::*;
pub use date::*;
pub use dateindex::*;
pub use decadeindex::*;
pub use difficultyepoch::*;
pub use dollars::*;
pub use feerate::*;
pub use halvingepoch::*;
pub use height::*;
pub use locktime::*;
pub use monthindex::*;
pub use ohlc::*;
pub use sats::*;
pub use timestamp::*;
@@ -52,4 +62,6 @@ pub use txversion::*;
pub use unit::*;
pub use vin::*;
pub use vout::*;
pub use weekindex::*;
pub use weight::*;
pub use yearindex::*;

48
crates/brk_core/src/structs/monthindex.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{fmt::Debug, ops::Add};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
Serialize,
Deserialize,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Monthindex(u16);
impl From<u16> for Monthindex {
fn from(value: u16) -> Self {
Self(value)
}
}
impl From<usize> for Monthindex {
fn from(value: usize) -> Self {
Self(value as u16)
}
}
impl From<Monthindex> for usize {
fn from(value: Monthindex) -> Self {
value.0 as usize
}
}
impl Add<usize> for Monthindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u16)
}
}

12
crates/brk_core/src/structs/sats.rs
View File

@@ -38,7 +38,7 @@ impl Sats {
}
impl Add for Sats {
type Output = Sats;
type Output = Self;
fn add(self, rhs: Sats) -> Self::Output {
Sats::from(self.0 + rhs.0)
}
@@ -51,8 +51,8 @@ impl AddAssign for Sats {
}
impl CheckedSub<Sats> for Sats {
fn checked_sub(self, rhs: Sats) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Sats::from)
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self::from)
}
}
@@ -63,21 +63,21 @@ impl SubAssign for Sats {
}
impl Mul<Sats> for Sats {
type Output = Sats;
type Output = Self;
fn mul(self, rhs: Sats) -> Self::Output {
Sats::from(self.0 * rhs.0)
}
}
impl Mul<u64> for Sats {
type Output = Sats;
type Output = Self;
fn mul(self, rhs: u64) -> Self::Output {
Sats::from(self.0 * rhs)
}
}
impl Mul<Height> for Sats {
type Output = Sats;
type Output = Self;
fn mul(self, rhs: Height) -> Self::Output {
Sats::from(self.0 * u64::from(rhs))
}

50
crates/brk_core/src/structs/timestamp.rs
View File

@@ -1,14 +1,36 @@
use std::ops::{Add, Div};
use derive_deref::Deref;
use jiff::{civil::date, tz::TimeZone};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
#[derive(
Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
Debug,
Deref,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Timestamp(u32);
impl Timestamp {
pub const ZERO: Self = Self(0);
pub fn new(timestamp: u32) -> Self {
Self(timestamp)
}
pub fn floor_seconds(self) -> Self {
let t = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC);
let d = jiff::civil::DateTime::from(t);
@@ -46,3 +68,29 @@ impl From<Timestamp> for bitcoin::locktime::absolute::Time {
bitcoin::locktime::absolute::Time::from_consensus(*value).unwrap()
}
}
impl From<usize> for Timestamp {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl CheckedSub<Timestamp> for Timestamp {
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self::from)
}
}
impl Div<usize> for Timestamp {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
Self(self.0 / rhs as u32)
}
}
impl Add for Timestamp {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self(self.0 + rhs.0)
}
}

4
crates/brk_core/src/structs/txinindex.rs
View File

@@ -61,8 +61,8 @@ impl AddAssign<Txinindex> for Txinindex {
}
impl CheckedSub<Txinindex> for Txinindex {
fn checked_sub(self, rhs: Txinindex) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Txinindex::from)
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self::from)
}
}

4
crates/brk_core/src/structs/txoutindex.rs
View File

@@ -67,8 +67,8 @@ impl AddAssign<Txoutindex> for Txoutindex {
}
impl CheckedSub<Txoutindex> for Txoutindex {
fn checked_sub(self, rhs: Txoutindex) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Txoutindex::from)
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self::from)
}
}

48
crates/brk_core/src/structs/weekindex.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{fmt::Debug, ops::Add};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
Serialize,
Deserialize,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Weekindex(u16);
impl From<u16> for Weekindex {
fn from(value: u16) -> Self {
Self(value)
}
}
impl From<usize> for Weekindex {
fn from(value: usize) -> Self {
Self(value as u16)
}
}
impl From<Weekindex> for usize {
fn from(value: Weekindex) -> Self {
value.0 as usize
}
}
impl Add<usize> for Weekindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u16)
}
}

48
crates/brk_core/src/structs/yearindex.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{fmt::Debug, ops::Add};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
Serialize,
Deserialize,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Yearindex(u8);
impl From<u8> for Yearindex {
fn from(value: u8) -> Self {
Self(value)
}
}
impl From<usize> for Yearindex {
fn from(value: usize) -> Self {
Self(value as u8)
}
}
impl From<Yearindex> for usize {
fn from(value: Yearindex) -> Self {
value.0 as usize
}
}
impl Add<usize> for Yearindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u8)
}
}

4
crates/brk_fetcher/examples/main.rs
View File

@@ -11,8 +11,8 @@ fn main() -> color_eyre::Result<()> {
dbg!(fetcher.get_date(Date::new(2025, 1, 1))?);
dbg!(fetcher.get_height(
881_000_u32.into(),
1740683986.into(),
Some(1740683000.into())
1740683986_u32.into(),
Some(1740683000_u32.into())
)?);
Ok(())

2
crates/brk_fetcher/src/lib.rs
View File

@@ -108,7 +108,7 @@ impl Fetcher {
let mut final_ohlc = OHLCCents::from(previous_ohlc.close);
let start = previous_timestamp.unwrap_or(Timestamp::from(0));
let start = previous_timestamp.unwrap_or(Timestamp::new(0));
let end = timestamp;
// Otherwise it's a re-org

7
crates/brk_indexer/examples/main.rs
View File

@@ -1,12 +1,9 @@
use std::path::Path;
use brk_core::{default_bitcoin_path, dot_brk_path};
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::{
Parser,
rpc::{self},
};
use brk_parser::{Parser, rpc};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

5
crates/brk_indexer/src/lib.rs
View File

@@ -7,7 +7,7 @@ use std::{
collections::BTreeMap,
path::{Path, PathBuf},
str::FromStr,
thread::{self},
thread,
};
use brk_core::{
@@ -31,6 +31,7 @@ pub use stores::*;
pub use vecs::*;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
const COLLISIONS_CHECKED_UP_TO: u32 = 870_000;
#[derive(Clone)]
pub struct Indexer {
@@ -138,7 +139,7 @@ impl Indexer {
idxs.height = height;
// Used to check rapidhash collisions
let check_collisions = self.check_collisions && height > Height::new(640_000);
let check_collisions = self.check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO);
let blockhash = BlockHash::from(blockhash);
let blockhash_prefix = BlockHashPrefix::from(&blockhash);

145
crates/brk_indexer/src/vecs/base.rs
View File

@@ -1,11 +1,14 @@
use std::{
cmp::Ordering,
fmt::Debug,
io,
ops::{Deref, DerefMut},
path::{Path, PathBuf},
};
use brk_vec::{Compressed, StoredIndex, StoredType, Version};
use brk_vec::{
AnyStorableVec, Compressed, Error, MAX_CACHE_SIZE, MAX_PAGE_SIZE, Result, StoredIndex,
StoredType, Value, Version,
};
use super::Height;
@@ -20,7 +23,16 @@ where
I: StoredIndex,
T: StoredType,
{
pub fn import(path: &Path, version: Version, compressed: Compressed) -> brk_vec::Result<Self> {
pub const SIZE_OF_T: usize = size_of::<T>();
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
pub fn forced_import(
path: &Path,
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
let mut vec = brk_vec::StorableVec::forced_import(path, version, compressed)?;
vec.enable_large_cache();
@@ -31,6 +43,76 @@ where
})
}
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
self.get_(index.to_usize()?)
}
fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
match self.vec.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
return Ok(self.vec.pushed().get(index).map(|v| Value::Ref(v)));
}
}
Err(Error::IndexTooHigh) => return Ok(None),
Err(Error::IndexTooLow) => {}
Err(error) => return Err(error),
}
let large_cache_len = self.vec.large_cache_len();
if large_cache_len != 0 {
let page_index = Self::index_to_page_index(index);
let last_index = self.vec.stored_len() - 1;
let max_page_index = Self::index_to_page_index(last_index);
let min_page_index = (max_page_index + 1) - large_cache_len;
if page_index >= min_page_index {
let values = self
.vec
.pages()
.unwrap()
.get(page_index - min_page_index)
.ok_or(Error::MmapsVecIsTooSmall)?
.get_or_init(|| self.vec.decode_page(page_index).unwrap());
return Ok(values.get(index)?.map(|v| Value::Ref(v)));
}
}
Ok(self.vec.read_(index)?.map(|v| Value::Owned(v)))
}
pub fn iter_from<F>(&mut self, index: I, f: F) -> Result<()>
where
F: FnMut((I, &T)) -> Result<()>,
{
self.vec.iter_from(index, f)
}
#[inline(always)]
fn index_to_page_index(index: usize) -> usize {
index / Self::PER_PAGE
}
#[inline]
pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
match self.vec.len().cmp(&index.to_usize()?) {
Ordering::Greater => {
// dbg!(len, index, &self.pathbuf);
// panic!();
Ok(())
}
Ordering::Equal => {
self.vec.push(value);
Ok(())
}
Ordering::Less => {
dbg!(index, value);
Err(Error::IndexTooHigh)
}
}
}
pub fn truncate_if_needed(&mut self, index: I, height: Height) -> brk_vec::Result<()> {
if self.height.is_none_or(|self_height| self_height != height) {
height.write(&self.path_height())?;
@@ -39,6 +121,36 @@ where
Ok(())
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
height.write(&self.path_height())?;
self.vec.flush()
}
pub fn vec(&self) -> &brk_vec::StorableVec<I, T> {
&self.vec
}
pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec<I, T> {
&mut self.vec
}
pub fn any_vec(&self) -> &dyn AnyStorableVec {
&self.vec
}
pub fn len(&self) -> usize {
self.vec.len()
}
pub fn is_empty(&self) -> bool {
self.vec.is_empty()
}
#[inline]
pub fn hasnt(&self, index: I) -> Result<bool> {
self.vec.has(index).map(|b| !b)
}
pub fn height(&self) -> brk_core::Result<Height> {
Height::try_from(self.path_height().as_path())
}
@@ -48,24 +160,19 @@ where
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
height.write(&self.path_height())?;
self.vec.flush()
}
}
impl<I, T> Deref for StorableVec<I, T> {
type Target = brk_vec::StorableVec<I, T>;
fn deref(&self) -> &Self::Target {
&self.vec
}
}
impl<I, T> DerefMut for StorableVec<I, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.vec
}
}
// impl<I, T> Deref for StorableVec<I, T> {
// type Target = brk_vec::StorableVec<I, T>;
// fn deref(&self) -> &Self::Target {
// &self.vec
// }
// }
// impl<I, T> DerefMut for StorableVec<I, T> {
// fn deref_mut(&mut self) -> &mut Self::Target {
// &mut self.vec
// }
// }
impl<I, T> Clone for StorableVec<I, T>
where
I: StoredIndex,

173
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -58,6 +58,7 @@ pub struct Vecs {
pub txindex_to_total_size: StorableVec<Txindex, usize>,
pub txindex_to_is_explicitly_rbf: StorableVec<Txindex, bool>,
pub txindex_to_txversion: StorableVec<Txindex, TxVersion>,
/// If txoutindex == Txoutindex MAX then is it's coinbase
pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex>,
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
pub txoutindex_to_value: StorableVec<Txoutindex, Sats>,
@@ -68,217 +69,217 @@ impl Vecs {
fs::create_dir_all(path)?;
Ok(Self {
addressindex_to_addresstype: StorableVec::import(
addressindex_to_addresstype: StorableVec::forced_import(
&path.join("addressindex_to_addresstype"),
Version::from(1),
compressed,
)?,
addressindex_to_addresstypeindex: StorableVec::import(
addressindex_to_addresstypeindex: StorableVec::forced_import(
&path.join("addressindex_to_addresstypeindex"),
Version::from(1),
compressed,
)?,
addressindex_to_height: StorableVec::import(
addressindex_to_height: StorableVec::forced_import(
&path.join("addressindex_to_height"),
Version::from(1),
compressed,
)?,
height_to_blockhash: StorableVec::import(
height_to_blockhash: StorableVec::forced_import(
&path.join("height_to_blockhash"),
Version::from(1),
Compressed::NO,
)?,
height_to_difficulty: StorableVec::import(
height_to_difficulty: StorableVec::forced_import(
&path.join("height_to_difficulty"),
Version::from(1),
compressed,
)?,
height_to_first_addressindex: StorableVec::import(
height_to_first_addressindex: StorableVec::forced_import(
&path.join("height_to_first_addressindex"),
Version::from(1),
compressed,
)?,
height_to_first_emptyindex: StorableVec::import(
height_to_first_emptyindex: StorableVec::forced_import(
&path.join("height_to_first_emptyindex"),
Version::from(1),
compressed,
)?,
height_to_first_multisigindex: StorableVec::import(
height_to_first_multisigindex: StorableVec::forced_import(
&path.join("height_to_first_multisigindex"),
Version::from(1),
compressed,
)?,
height_to_first_opreturnindex: StorableVec::import(
height_to_first_opreturnindex: StorableVec::forced_import(
&path.join("height_to_first_opreturnindex"),
Version::from(1),
compressed,
)?,
height_to_first_pushonlyindex: StorableVec::import(
height_to_first_pushonlyindex: StorableVec::forced_import(
&path.join("height_to_first_pushonlyindex"),
Version::from(1),
compressed,
)?,
height_to_first_txindex: StorableVec::import(
height_to_first_txindex: StorableVec::forced_import(
&path.join("height_to_first_txindex"),
Version::from(1),
compressed,
)?,
height_to_first_txinindex: StorableVec::import(
height_to_first_txinindex: StorableVec::forced_import(
&path.join("height_to_first_txinindex"),
Version::from(1),
compressed,
)?,
height_to_first_txoutindex: StorableVec::import(
height_to_first_txoutindex: StorableVec::forced_import(
&path.join("height_to_first_txoutindex"),
Version::from(1),
compressed,
)?,
height_to_first_unknownindex: StorableVec::import(
height_to_first_unknownindex: StorableVec::forced_import(
&path.join("height_to_first_unkownindex"),
Version::from(1),
compressed,
)?,
height_to_first_p2pk33index: StorableVec::import(
height_to_first_p2pk33index: StorableVec::forced_import(
&path.join("height_to_first_p2pk33index"),
Version::from(1),
compressed,
)?,
height_to_first_p2pk65index: StorableVec::import(
height_to_first_p2pk65index: StorableVec::forced_import(
&path.join("height_to_first_p2pk65index"),
Version::from(1),
compressed,
)?,
height_to_first_p2pkhindex: StorableVec::import(
height_to_first_p2pkhindex: StorableVec::forced_import(
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
compressed,
)?,
height_to_first_p2shindex: StorableVec::import(
height_to_first_p2shindex: StorableVec::forced_import(
&path.join("height_to_first_p2shindex"),
Version::from(1),
compressed,
)?,
height_to_first_p2trindex: StorableVec::import(
height_to_first_p2trindex: StorableVec::forced_import(
&path.join("height_to_first_p2trindex"),
Version::from(1),
compressed,
)?,
height_to_first_p2wpkhindex: StorableVec::import(
height_to_first_p2wpkhindex: StorableVec::forced_import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
compressed,
)?,
height_to_first_p2wshindex: StorableVec::import(
height_to_first_p2wshindex: StorableVec::forced_import(
&path.join("height_to_first_p2wshindex"),
Version::from(1),
compressed,
)?,
height_to_size: StorableVec::import(
height_to_size: StorableVec::forced_import(
&path.join("height_to_size"),
Version::from(1),
compressed,
)?,
height_to_timestamp: StorableVec::import(
height_to_timestamp: StorableVec::forced_import(
&path.join("height_to_timestamp"),
Version::from(1),
compressed,
)?,
height_to_weight: StorableVec::import(
height_to_weight: StorableVec::forced_import(
&path.join("height_to_weight"),
Version::from(1),
compressed,
)?,
p2pk33index_to_p2pk33addressbytes: StorableVec::import(
p2pk33index_to_p2pk33addressbytes: StorableVec::forced_import(
&path.join("p2pk33index_to_p2pk33addressbytes"),
Version::from(1),
Compressed::NO,
)?,
p2pk65index_to_p2pk65addressbytes: StorableVec::import(
p2pk65index_to_p2pk65addressbytes: StorableVec::forced_import(
&path.join("p2pk65index_to_p2pk65addressbytes"),
Version::from(1),
Compressed::NO,
)?,
p2pkhindex_to_p2pkhaddressbytes: StorableVec::import(
p2pkhindex_to_p2pkhaddressbytes: StorableVec::forced_import(
&path.join("p2pkhindex_to_p2pkhaddressbytes"),
Version::from(1),
Compressed::NO,
)?,
p2shindex_to_p2shaddressbytes: StorableVec::import(
p2shindex_to_p2shaddressbytes: StorableVec::forced_import(
&path.join("p2shindex_to_p2shaddressbytes"),
Version::from(1),
Compressed::NO,
)?,
p2trindex_to_p2traddressbytes: StorableVec::import(
p2trindex_to_p2traddressbytes: StorableVec::forced_import(
&path.join("p2trindex_to_p2traddressbytes"),
Version::from(1),
Compressed::NO,
)?,
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import(
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::forced_import(
&path.join("p2wpkhindex_to_p2wpkhaddressbytes"),
Version::from(1),
Compressed::NO,
)?,
p2wshindex_to_p2wshaddressbytes: StorableVec::import(
p2wshindex_to_p2wshaddressbytes: StorableVec::forced_import(
&path.join("p2wshindex_to_p2wshaddressbytes"),
Version::from(1),
Compressed::NO,
)?,
txindex_to_first_txinindex: StorableVec::import(
txindex_to_first_txinindex: StorableVec::forced_import(
&path.join("txindex_to_first_txinindex"),
Version::from(1),
compressed,
)?,
txindex_to_first_txoutindex: StorableVec::import(
txindex_to_first_txoutindex: StorableVec::forced_import(
&path.join("txindex_to_first_txoutindex"),
Version::from(1),
Compressed::NO,
)?,
txindex_to_height: StorableVec::import(
txindex_to_height: StorableVec::forced_import(
&path.join("txindex_to_height"),
Version::from(1),
compressed,
)?,
txindex_to_locktime: StorableVec::import(
txindex_to_locktime: StorableVec::forced_import(
&path.join("txindex_to_locktime"),
Version::from(1),
compressed,
)?,
txindex_to_txid: StorableVec::import(
txindex_to_txid: StorableVec::forced_import(
&path.join("txindex_to_txid"),
Version::from(1),
Compressed::NO,
)?,
txindex_to_base_size: StorableVec::import(
txindex_to_base_size: StorableVec::forced_import(
&path.join("txindex_to_base_size"),
Version::from(1),
compressed,
)?,
txindex_to_total_size: StorableVec::import(
txindex_to_total_size: StorableVec::forced_import(
&path.join("txindex_to_total_size"),
Version::from(1),
compressed,
)?,
txindex_to_is_explicitly_rbf: StorableVec::import(
txindex_to_is_explicitly_rbf: StorableVec::forced_import(
&path.join("txindex_to_is_explicitly_rbf"),
Version::from(1),
compressed,
)?,
txindex_to_txversion: StorableVec::import(
txindex_to_txversion: StorableVec::forced_import(
&path.join("txindex_to_txversion"),
Version::from(1),
compressed,
)?,
txinindex_to_txoutindex: StorableVec::import(
txinindex_to_txoutindex: StorableVec::forced_import(
&path.join("txinindex_to_txoutindex"),
Version::from(1),
compressed,
)?,
txoutindex_to_addressindex: StorableVec::import(
txoutindex_to_addressindex: StorableVec::forced_import(
&path.join("txoutindex_to_addressindex"),
Version::from(1),
compressed,
)?,
txoutindex_to_value: StorableVec::import(
txoutindex_to_value: StorableVec::forced_import(
&path.join("txoutindex_to_value"),
Version::from(1),
compressed,
@@ -496,49 +497,49 @@ impl Vecs {
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
vec![
&*self.addressindex_to_addresstype,
&*self.addressindex_to_addresstypeindex,
&*self.addressindex_to_height,
&*self.height_to_blockhash,
&*self.height_to_difficulty,
&*self.height_to_first_addressindex,
&*self.height_to_first_emptyindex,
&*self.height_to_first_multisigindex,
&*self.height_to_first_opreturnindex,
&*self.height_to_first_pushonlyindex,
&*self.height_to_first_txindex,
&*self.height_to_first_txinindex,
&*self.height_to_first_txoutindex,
&*self.height_to_first_unknownindex,
&*self.height_to_first_p2pk33index,
&*self.height_to_first_p2pk65index,
&*self.height_to_first_p2pkhindex,
&*self.height_to_first_p2shindex,
&*self.height_to_first_p2trindex,
&*self.height_to_first_p2wpkhindex,
&*self.height_to_first_p2wshindex,
&*self.height_to_size,
&*self.height_to_timestamp,
&*self.height_to_weight,
&*self.p2pk33index_to_p2pk33addressbytes,
&*self.p2pk65index_to_p2pk65addressbytes,
&*self.p2pkhindex_to_p2pkhaddressbytes,
&*self.p2shindex_to_p2shaddressbytes,
&*self.p2trindex_to_p2traddressbytes,
&*self.p2wpkhindex_to_p2wpkhaddressbytes,
&*self.p2wshindex_to_p2wshaddressbytes,
&*self.txindex_to_first_txinindex,
&*self.txindex_to_first_txoutindex,
&*self.txindex_to_height,
&*self.txindex_to_locktime,
&*self.txindex_to_txid,
&*self.txindex_to_base_size,
&*self.txindex_to_total_size,
&*self.txindex_to_is_explicitly_rbf,
&*self.txindex_to_txversion,
&*self.txinindex_to_txoutindex,
&*self.txoutindex_to_addressindex,
&*self.txoutindex_to_value,
self.addressindex_to_addresstype.any_vec(),
self.addressindex_to_addresstypeindex.any_vec(),
self.addressindex_to_height.any_vec(),
self.height_to_blockhash.any_vec(),
self.height_to_difficulty.any_vec(),
self.height_to_first_addressindex.any_vec(),
self.height_to_first_emptyindex.any_vec(),
self.height_to_first_multisigindex.any_vec(),
self.height_to_first_opreturnindex.any_vec(),
self.height_to_first_pushonlyindex.any_vec(),
self.height_to_first_txindex.any_vec(),
self.height_to_first_txinindex.any_vec(),
self.height_to_first_txoutindex.any_vec(),
self.height_to_first_unknownindex.any_vec(),
self.height_to_first_p2pk33index.any_vec(),
self.height_to_first_p2pk65index.any_vec(),
self.height_to_first_p2pkhindex.any_vec(),
self.height_to_first_p2shindex.any_vec(),
self.height_to_first_p2trindex.any_vec(),
self.height_to_first_p2wpkhindex.any_vec(),
self.height_to_first_p2wshindex.any_vec(),
self.height_to_size.any_vec(),
self.height_to_timestamp.any_vec(),
self.height_to_weight.any_vec(),
self.p2pk33index_to_p2pk33addressbytes.any_vec(),
self.p2pk65index_to_p2pk65addressbytes.any_vec(),
self.p2pkhindex_to_p2pkhaddressbytes.any_vec(),
self.p2shindex_to_p2shaddressbytes.any_vec(),
self.p2trindex_to_p2traddressbytes.any_vec(),
self.p2wpkhindex_to_p2wpkhaddressbytes.any_vec(),
self.p2wshindex_to_p2wshaddressbytes.any_vec(),
self.txindex_to_first_txinindex.any_vec(),
self.txindex_to_first_txoutindex.any_vec(),
self.txindex_to_height.any_vec(),
self.txindex_to_locktime.any_vec(),
self.txindex_to_txid.any_vec(),
self.txindex_to_base_size.any_vec(),
self.txindex_to_total_size.any_vec(),
self.txindex_to_is_explicitly_rbf.any_vec(),
self.txindex_to_txversion.any_vec(),
self.txinindex_to_txoutindex.any_vec(),
self.txoutindex_to_addressindex.any_vec(),
self.txoutindex_to_value.any_vec(),
]
}

9
crates/brk_parser/src/lib.rs
View File

@@ -3,14 +3,7 @@
#![doc = include_str!("../examples/main.rs")]
#![doc = "```"]
use std::{
cmp::Ordering,
collections::BTreeMap,
fs::{self},
ops::ControlFlow,
path::PathBuf,
thread,
};
use std::{cmp::Ordering, collections::BTreeMap, fs, ops::ControlFlow, path::PathBuf, thread};
use bitcoin::{Block, BlockHash};
use bitcoincore_rpc::RpcApi;

4
crates/brk_server/Cargo.toml
View File

@@ -21,8 +21,8 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.58.1", features = ["codegen", "minifier"] }
oxc = { version = "0.60.0", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { version = "1.44.1", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full"] }
zip = "2.2.3"
zip = "2.4.1"

6
crates/brk_server/src/files/file.rs
View File

@@ -1,8 +1,4 @@
use std::{
fs::{self},
path::Path,
time::Instant,
};
use std::{fs, path::Path, time::Instant};
use axum::{
body::Body,

10
crates/brk_vec/examples/main.rs
View File

@@ -24,12 +24,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
StorableVec::forced_import(Path::new("./vec"), Version::from(1), Compressed::YES)?;
dbg!(vec.get(0)?);
dbg!(vec.read(0)?);
dbg!(vec.read(1)?);
dbg!(vec.read(2)?);
dbg!(vec.read(20)?);
dbg!(vec.get(0)?);
dbg!(vec.get(1)?);
dbg!(vec.get(2)?);
dbg!(vec.get(20)?);
dbg!(vec.read(0)?);
dbg!(vec.get(20)?);
dbg!(vec.get(0)?);
vec.push(21);
vec.push(22);

113
crates/brk_vec/src/lib.rs
View File

@@ -4,7 +4,6 @@
#![doc = "```"]
use std::{
cmp::Ordering,
fs::{self, File, OpenOptions},
io::{self, Read, Seek, SeekFrom, Write},
marker::PhantomData,
@@ -29,7 +28,7 @@ pub use traits::*;
const ONE_KIB: usize = 1024;
pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
#[allow(private_interfaces)]
#[derive(Debug)]
@@ -83,49 +82,11 @@ where
}
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
self.get_(Self::i_to_usize(index)?)
}
fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
return Ok(self.pushed().get(index).map(|v| Value::Ref(v)));
}
}
Err(Error::IndexTooHigh) => return Ok(None),
Err(Error::IndexTooLow) => {}
Err(error) => return Err(error),
}
let large_cache_len = self.large_cache_len();
if large_cache_len != 0 {
let page_index = Self::index_to_page_index(index);
let last_index = self.stored_len() - 1;
let max_page_index = Self::index_to_page_index(last_index);
let min_page_index = (max_page_index + 1) - large_cache_len;
if page_index >= min_page_index {
let values = self
.pages()
.unwrap()
.get(page_index - min_page_index)
.ok_or(Error::MmapsVecIsTooSmall)?
.get_or_init(|| self.decode_page(page_index).unwrap());
return Ok(values.get(index)?.map(|v| Value::Ref(v)));
}
}
Ok(self.open_then_read_(index)?.map(|v| Value::Owned(v)))
}
#[inline]
pub fn read(&mut self, index: I) -> Result<Option<&T>> {
self.read_(Self::i_to_usize(index)?)
pub fn get(&mut self, index: I) -> Result<Option<&T>> {
self.get_(index.to_usize()?)
}
#[inline]
pub fn read_(&mut self, index: usize) -> Result<Option<&T>> {
pub fn get_(&mut self, index: usize) -> Result<Option<&T>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
@@ -147,18 +108,18 @@ where
self.page().unwrap().1.get(index)
}
pub fn read_last(&mut self) -> Result<Option<&T>> {
pub fn get_last(&mut self) -> Result<Option<&T>> {
let len = self.len();
if len == 0 {
return Ok(None);
}
self.read_(len - 1)
self.get_(len - 1)
}
pub fn open_then_read(&self, index: I) -> Result<Option<T>> {
self.open_then_read_(Self::i_to_usize(index)?)
pub fn read(&self, index: I) -> Result<Option<T>> {
self.read_(index.to_usize()?)
}
fn open_then_read_(&self, index: usize) -> Result<Option<T>> {
pub fn read_(&self, index: usize) -> Result<Option<T>> {
Ok(match self {
Self::Raw { .. } => {
let mut file = self.open_file()?;
@@ -193,7 +154,7 @@ where
let stored_len = I::from(self.stored_len());
while index < stored_len {
let v = self.read(index)?.unwrap();
let v = self.get(index)?.unwrap();
f((index, v))?;
index = index + 1;
}
@@ -212,7 +173,7 @@ where
let stored_len = I::from(self.stored_len());
while index < stored_len {
let v = self.read(index)?.unwrap().clone();
let v = self.get(index)?.unwrap().clone();
f((index, v, self))?;
index = index + 1;
}
@@ -265,7 +226,7 @@ where
Ok(values)
}
fn decode_page(&self, page_index: usize) -> Result<Values<T>> {
pub fn decode_page(&self, page_index: usize) -> Result<Values<T>> {
Self::decode_page_(
self.stored_len(),
page_index,
@@ -327,26 +288,7 @@ where
#[inline]
pub fn push(&mut self, value: T) {
self.mut_pushed().push(value)
}
#[inline]
pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
match self.len().cmp(&Self::i_to_usize(index)?) {
Ordering::Greater => {
// dbg!(len, index, &self.pathbuf);
// panic!();
Ok(())
}
Ordering::Equal => {
self.mut_pushed().push(value);
Ok(())
}
Ordering::Less => {
dbg!(index, value);
Err(Error::IndexTooHigh)
}
}
self.mut_base().pushed.push(value)
}
pub fn flush(&mut self) -> io::Result<()> {
@@ -464,7 +406,7 @@ where
}
pub fn truncate_if_needed(&mut self, index: I) -> Result<()> {
let index = Self::i_to_usize(index)?;
let index = index.to_usize()?;
if index >= self.stored_len() {
return Ok(());
@@ -580,7 +522,7 @@ where
}
}
fn large_cache_len(&self) -> usize {
pub fn large_cache_len(&self) -> usize {
self.pages().map_or(0, |v| v.len())
}
@@ -601,12 +543,7 @@ where
}
#[inline]
pub fn i_to_usize(index: I) -> Result<usize> {
index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)
}
#[inline]
fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
pub fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
let stored_len = self.stored_len();
if index >= stored_len {
@@ -662,7 +599,7 @@ where
}
#[inline]
fn pages(&self) -> Option<&Vec<OnceLock<Values<T>>>> {
pub fn pages(&self) -> Option<&Vec<OnceLock<Values<T>>>> {
self.base().pages.as_ref()
}
@@ -682,7 +619,7 @@ where
#[inline]
pub fn has(&self, index: I) -> Result<bool> {
Ok(self.has_(Self::i_to_usize(index)?))
Ok(self.has_(index.to_usize()?))
}
#[inline]
fn has_(&self, index: usize) -> bool {
@@ -690,20 +627,10 @@ where
}
#[inline]
pub fn hasnt(&self, index: I) -> Result<bool> {
self.has(index).map(|b| !b)
}
#[inline]
fn pushed(&self) -> &Vec<T> {
pub fn pushed(&self) -> &Vec<T> {
&self.base().pushed
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
&mut self.mut_base().pushed
}
#[inline]
pub fn pushed_len(&self) -> usize {
self.pushed().len()
@@ -715,7 +642,7 @@ where
}
#[inline]
fn stored_len(&self) -> usize {
pub fn stored_len(&self) -> usize {
*self.base().stored_len
}

7
crates/brk_vec/src/structs/compressed.rs
View File

@@ -1,9 +1,4 @@
use std::{
fs,
io::{self},
ops::Deref,
path::Path,
};
use std::{fs, io, ops::Deref, path::Path};
use crate::{Error, Result};

6
crates/brk_vec/src/structs/version.rs
View File

@@ -9,7 +9,9 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Error, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout)]
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout,
)]
pub struct Version(u32);
impl Version {
@@ -54,7 +56,7 @@ impl TryFrom<&Path> for Version {
}
impl Add<Version> for Version {
type Output = Version;
type Output = Self;
fn add(self, rhs: Version) -> Self::Output {
Self(self.0 + rhs.0)
}

12
crates/brk_vec/src/traits/stored_index.rs
View File

@@ -1,5 +1,7 @@
use std::{fmt::Debug, ops::Add};
use crate::{Error, Result};
pub trait StoredIndex
where
Self: Debug
@@ -16,8 +18,10 @@ where
+ Send
+ Sync,
{
fn to_usize(self) -> Result<usize>;
}
impl<I> StoredIndex for I where
impl<I> StoredIndex for I
where
I: Debug
+ Default
+ Copy
@@ -30,6 +34,10 @@ impl<I> StoredIndex for I where
+ From<usize>
+ Add<usize, Output = Self>
+ Send
+ Sync
+ Sync,
{
#[inline(always)]
fn to_usize(self) -> Result<usize> {
self.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)
}
}