Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: one big snapshot

Browse Source
This commit is contained in:
nym21
2025-08-02 16:59:22 +02:00
parent aa8b47a3dd
commit f7aa9424db
252 changed files with 6283 additions and 5264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.cargo/config.toml Normal file
View File

@@ -0,0 +1,14 @@
# Release profile: all targets get native CPU
[profile.release.target.'cfg()']
rustflags = ["-C", "target-cpu=native"]
# Release profile: x86_64 gets native CPU + extra features (overrides the above)
[profile.release.target.'cfg(target_arch = "x86_64")']
rustflags = [
"-C", "target-cpu=native",
"-C", "target-feature=+bmi1,+bmi2,+avx2"
]
# Dist profile: x86_64 gets only extra features (no native CPU)
[profile.dist.target.'cfg(target_arch = "x86_64")']
rustflags = ["-C", "target-feature=+bmi1,+bmi2,+avx2"]

324
Cargo.lock generated
View File

@@ -18,14 +18,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aes"
version = "0.8.4"
name = "ahash"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"cipher",
"cpufeatures",
"getrandom 0.3.3",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
@@ -319,6 +321,12 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
[[package]]
name = "better_io"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92fde17f91e7ba10b2a07f8dff29530b77144894bc6ae850fbc66e1276af0d28"
[[package]]
name = "bincode"
version = "2.0.1"
@@ -476,8 +484,7 @@ dependencies = [
"brk_bundler",
"brk_cli",
"brk_computer",
"brk_core",
"brk_exit",
"brk_error",
"brk_fetcher",
"brk_indexer",
"brk_interface",
@@ -486,6 +493,7 @@ dependencies = [
"brk_parser",
"brk_server",
"brk_store",
"brk_structs",
"brk_vecs",
]
@@ -506,8 +514,6 @@ version = "0.0.83"
dependencies = [
"bitcoincore-rpc",
"brk_computer",
"brk_core",
"brk_exit",
"brk_fetcher",
"brk_indexer",
"brk_logger",
@@ -529,14 +535,13 @@ version = "0.0.83"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"brk_core",
"brk_exit",
"brk_error",
"brk_fetcher",
"brk_indexer",
"brk_logger",
"brk_parser",
"brk_structs",
"brk_vecs",
"color-eyre",
"derive_deref",
"log",
"rayon",
@@ -546,42 +551,24 @@ dependencies = [
]
[[package]]
name = "brk_core"
name = "brk_error"
version = "0.0.83"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"byteview",
"derive_deref",
"brk_vecs",
"fjall",
"jiff",
"log",
"rapidhash",
"rlimit",
"serde",
"serde_bytes",
"serde_json",
"minreq",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "brk_exit"
version = "0.0.83"
dependencies = [
"brk_logger",
"ctrlc",
"log",
"parking_lot",
]
[[package]]
name = "brk_fetcher"
version = "0.0.83"
dependencies = [
"brk_core",
"brk_error",
"brk_logger",
"color-eyre",
"brk_structs",
"log",
"minreq",
"serde_json",
@@ -593,13 +580,12 @@ version = "0.0.83"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"brk_core",
"brk_exit",
"brk_error",
"brk_logger",
"brk_parser",
"brk_store",
"brk_structs",
"brk_vecs",
"color-eyre",
"fjall",
"log",
"rayon",
@@ -610,10 +596,10 @@ name = "brk_interface"
version = "0.0.83"
dependencies = [
"brk_computer",
"brk_core",
"brk_error",
"brk_indexer",
"brk_structs",
"brk_vecs",
"color-eyre",
"derive_deref",
"schemars 1.0.4",
"serde",
@@ -626,10 +612,10 @@ dependencies = [
name = "brk_logger"
version = "0.0.83"
dependencies = [
"color-eyre",
"env_logger",
"jiff",
"log",
"owo-colors",
]
[[package]]
@@ -648,7 +634,7 @@ version = "0.0.83"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"brk_core",
"brk_structs",
"crossbeam",
"derive_deref",
"rayon",
@@ -1019,8 +1005,7 @@ dependencies = [
"bitcoincore-rpc",
"brk_bundler",
"brk_computer",
"brk_core",
"brk_exit",
"brk_error",
"brk_fetcher",
"brk_indexer",
"brk_interface",
@@ -1030,10 +1015,11 @@ dependencies = [
"brk_vecs",
"clap",
"clap_derive",
"color-eyre",
"jiff",
"log",
"minreq",
"owo-colors",
"quick_cache",
"serde",
"serde_json",
"tokio",
@@ -1046,7 +1032,8 @@ dependencies = [
name = "brk_store"
version = "0.0.83"
dependencies = [
"brk_core",
"brk_error",
"brk_structs",
"byteview",
"fjall",
"log",
@@ -1064,24 +1051,50 @@ dependencies = [
"serde",
]
[[package]]
name = "brk_structs"
version = "0.0.83"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"brk_error",
"brk_vecs",
"byteview",
"derive_deref",
"fjall",
"jiff",
"rapidhash",
"serde",
"serde_bytes",
"serde_json",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "brk_vecs"
version = "0.0.83"
dependencies = [
"brk_core",
"brk_exit",
"clap",
"clap_derive",
"brk_vecs_macros",
"ctrlc",
"libc",
"log",
"memmap2",
"parking_lot",
"pco",
"rayon",
"serde",
"serde_json",
"zerocopy",
"zerocopy-derive",
"zstd",
]
[[package]]
name = "brk_vecs_macros"
version = "0.0.83"
dependencies = [
"quote",
"syn 2.0.104",
]
[[package]]
@@ -1138,15 +1151,6 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]]
name = "bzip2"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff"
dependencies = [
"libbz2-rs-sys",
]
[[package]]
name = "castaway"
version = "0.2.4"
@@ -1158,9 +1162,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.30"
version = "1.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2"
dependencies = [
"jobserver",
"libc",
@@ -1205,30 +1209,20 @@ dependencies = [
"windows-link",
]
[[package]]
name = "cipher"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common",
"inout",
]
[[package]]
name = "clap"
version = "4.5.41"
version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.5.41"
version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966"
dependencies = [
"anstream",
"anstyle",
@@ -1434,6 +1428,12 @@ version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.1.6"
@@ -1547,12 +1547,6 @@ dependencies = [
"parking_lot_core",
]
[[package]]
name = "deflate64"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
[[package]]
name = "deranged"
version = "0.4.0"
@@ -1614,7 +1608,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
@@ -1623,6 +1616,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]]
name = "dtype_dispatch"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3a5ccdfd6c5e7e2fea9c5cf256f2a08216047fab19c621c3da64e9ae4a1462d"
[[package]]
name = "dunce"
version = "1.0.5"
@@ -1944,6 +1943,16 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
[[package]]
name = "half"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -1994,15 +2003,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
[[package]]
name = "hmac"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
"digest",
]
[[package]]
name = "http"
version = "1.3.1"
@@ -2171,15 +2171,6 @@ dependencies = [
"libc",
]
[[package]]
name = "inout"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"generic-array",
]
[[package]]
name = "interval-heap"
version = "0.0.5"
@@ -2329,43 +2320,17 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libbz2-rs-sys"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb"
[[package]]
name = "libc"
version = "0.2.174"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
[[package]]
name = "liblzma"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0791ab7e08ccc8e0ce893f6906eb2703ed8739d8e89b57c0714e71bad09024c8"
dependencies = [
"liblzma-sys",
]
[[package]]
name = "liblzma-sys"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "libredox"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "360e552c93fa0e8152ab463bc4c4837fce76a225df11dfaeea66c313de5e61f7"
checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
dependencies = [
"bitflags 2.9.1",
"libc",
@@ -2673,9 +2638,9 @@ dependencies = [
[[package]]
name = "oxc-browserslist"
version = "2.0.11"
version = "2.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "509ae58826553dbd720886b6a77e7bb2a953d28a52728a74cc1fe296d66fa9a2"
checksum = "e05d19022f54d3e0b8b1679c80f02d140e95e4308385eb247ba3168c299c81bb"
dependencies = [
"bincode",
"nom",
@@ -2688,9 +2653,9 @@ dependencies = [
[[package]]
name = "oxc-miette"
version = "2.3.2"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8f905f1293ce969f8b6f08f2f4e8a0fbfd26dec42a30ea98b436d2faf01f50f"
checksum = "31cfb121c9d3e0f9082856927f5cff9594279c91b544f4436e4bc971563caa60"
dependencies = [
"cfg-if",
"owo-colors",
@@ -2702,9 +2667,9 @@ dependencies = [
[[package]]
name = "oxc-miette-derive"
version = "2.3.2"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd092b10278cef8cc76dad34116988c2f0ca34bf9082de5513bacd95f4553d0"
checksum = "a6eabb57f935b454fbe0552ea0abaaf9eb0019b5fa05a7bbe7efd5bd8c765085"
dependencies = [
"proc-macro2",
"quote",
@@ -3198,13 +3163,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
[[package]]
name = "pbkdf2"
version = "0.12.2"
name = "pco"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
checksum = "ab068b64f2c6f074cbdcafc80ebd83a27da92a3848deba2fabc21eba6691fc65"
dependencies = [
"digest",
"hmac",
"better_io",
"dtype_dispatch",
"half",
"rand_xoshiro",
]
[[package]]
@@ -3347,12 +3314,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppmd-rust"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b"
[[package]]
name = "ppv-lite86"
version = "0.2.21"
@@ -3399,8 +3360,10 @@ version = "0.6.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8565e62e02af316570d4b492f17af1481d6c07cea60f4e7edd71700da5052ba9"
dependencies = [
"ahash",
"equivalent",
"hashbrown 0.15.4",
"parking_lot",
]
[[package]]
@@ -3487,6 +3450,15 @@ dependencies = [
"getrandom 0.3.3",
]
[[package]]
name = "rand_xoshiro"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
dependencies = [
"rand_core 0.6.4",
]
[[package]]
name = "rapidhash"
version = "2.0.2"
@@ -3610,15 +3582,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rlimit"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a"
dependencies = [
"libc",
]
[[package]]
name = "rolldown-ariadne"
version = "0.5.2"
@@ -3852,9 +3815,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.141"
version = "1.0.142"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
dependencies = [
"indexmap 2.10.0",
"itoa",
@@ -4035,12 +3998,6 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "sugar_path"
version = "1.2.0"
@@ -4223,9 +4180,9 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.47.0"
version = "1.47.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35"
checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
dependencies = [
"backtrace",
"bytes",
@@ -4276,9 +4233,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.9.3"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e06723639aaded957e5a80be250c1f82f274b9d23ebb4d94163668470623461c"
checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1"
dependencies = [
"indexmap 2.10.0",
"serde",
@@ -5011,51 +4968,18 @@ dependencies = [
"syn 2.0.104",
]
[[package]]
name = "zeroize"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
dependencies = [
"zeroize_derive",
]
[[package]]
name = "zeroize_derive"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
]
[[package]]
name = "zip"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aed4ac33e8eb078c89e6cbb1d5c4c7703ec6d299fc3e7c3695af8f8b423468b"
dependencies = [
"aes",
"arbitrary",
"bzip2",
"constant_time_eq",
"crc32fast",
"deflate64",
"flate2",
"getrandom 0.3.3",
"hmac",
"indexmap 2.10.0",
"liblzma",
"memchr",
"pbkdf2",
"ppmd-rust",
"sha1",
"time",
"zeroize",
"zopfli",
"zstd",
]
[[package]]

19
Cargo.toml
View File

@@ -28,8 +28,8 @@ bitcoincore-rpc = "0.19.0"
brk_bundler = { version = "0.0.83", path = "crates/brk_bundler" }
brk_cli = { version = "0.0.83", path = "crates/brk_cli" }
brk_computer = { version = "0.0.83", path = "crates/brk_computer" }
brk_core = { version = "0.0.83", path = "crates/brk_core" }
brk_exit = { version = "0.0.83", path = "crates/brk_exit" }
brk_structs = { version = "0.0.83", path = "crates/brk_structs" }
brk_error = { version = "0.0.83", path = "crates/brk_error" }
brk_fetcher = { version = "0.0.83", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.0.83", path = "crates/brk_indexer" }
brk_interface = { version = "0.0.83", path = "crates/brk_interface" }
@@ -39,23 +39,24 @@ brk_parser = { version = "0.0.83", path = "crates/brk_parser" }
brk_server = { version = "0.0.83", path = "crates/brk_server" }
brk_store = { version = "0.0.83", path = "crates/brk_store" }
brk_vecs = { version = "0.0.83", path = "crates/brk_vecs" }
brk_vecs_macros = { version = "0.0.83", path = "crates/brk_vecs_macros" }
byteview = "=0.6.1"
clap = { version = "4.5.41", features = ["string"] }
clap = { version = "4.5.42", features = ["string"] }
clap_derive = "4.5.41"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall = "2.11.2"
jiff = "0.2.15"
log = { version = "0.4.27" }
log = "0.4.27"
minreq = { version = "2.14.0", features = ["https", "serde_json"] }
owo-colors = "4.2.2"
parking_lot = "0.12.4"
rayon = "1.10.0"
serde = { version = "1.0.219" }
serde = "1.0.219"
serde_bytes = "0.11.17"
serde_derive = "1.0.219"
serde_json = { version = "1.0.141", features = ["float_roundtrip"] }
tokio = { version = "1.47.0", features = ["rt-multi-thread"] }
zerocopy = { version = "0.8.26" }
serde_json = { version = "1.0.142", features = ["float_roundtrip"] }
tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
zerocopy = "0.8.26"
zerocopy-derive = "0.8.26"
[workspace.metadata.release]

9
README.md
View File

@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network.
@@ -63,8 +57,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
- [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates
- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a Bitcoin Research Kit instance
- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer
- [`brk_core`](https://crates.io/crates/brk_core): The Core (Structs and Errors) of the Bitcoin Research Kit
- [`brk_exit`](https://crates.io/crates/brk_exit): An exit blocker built on top of ctrlc
- [`brk_structs`](https://crates.io/crates/brk_structs): The Core (Structs and Errors) of the Bitcoin Research Kit
- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher
- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser
- [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit

18
crates/brk/Cargo.toml
View File

@@ -11,47 +11,47 @@ version.workspace = true
[features]
full = [
"bundler",
"core",
"computer",
"exit",
"error",
"fetcher",
"indexer",
"interface",
"logger",
"mcp",
"parser",
"interface",
"server",
"store",
"structs",
"vecs",
]
bundler = ["brk_bundler"]
core = ["brk_core"]
computer = ["brk_computer"]
exit = ["brk_exit"]
error = ["brk_error"]
fetcher = ["brk_fetcher"]
indexer = ["brk_indexer"]
interface = ["brk_interface"]
logger = ["brk_logger"]
mcp = ["brk_mcp"]
parser = ["brk_parser"]
interface = ["brk_interface"]
server = ["brk_server"]
store = ["brk_store"]
structs = ["brk_structs"]
vecs = ["brk_vecs"]
[dependencies]
brk_bundler = { workspace = true, optional = true }
brk_cli = { workspace = true }
brk_core = { workspace = true, optional = true }
brk_computer = { workspace = true, optional = true }
brk_exit = { workspace = true, optional = true }
brk_error = { workspace = true, optional = true }
brk_fetcher = { workspace = true, optional = true }
brk_indexer = { workspace = true, optional = true }
brk_interface = { workspace = true, optional = true }
brk_logger = { workspace = true, optional = true }
brk_mcp = { workspace = true, optional = true }
brk_parser = { workspace = true, optional = true }
brk_interface = { workspace = true, optional = true }
brk_server = { workspace = true, optional = true }
brk_store = { workspace = true, optional = true }
brk_structs = { workspace = true, optional = true }
brk_vecs = { workspace = true, optional = true }
[package.metadata.docs.rs]

8
crates/brk/src/lib.rs
View File

@@ -7,17 +7,17 @@ pub use brk_bundler as bundler;
#[doc(inline)]
pub use brk_cli as cli;
#[cfg(feature = "core")]
#[cfg(feature = "structs")]
#[doc(inline)]
pub use brk_core as core;
pub use brk_structs as structs;
#[cfg(feature = "computer")]
#[doc(inline)]
pub use brk_computer as computer;
#[cfg(feature = "exit")]
#[cfg(feature = "error")]
#[doc(inline)]
pub use brk_exit as exit;
pub use brk_error as error;
#[cfg(feature = "fetcher")]
#[doc(inline)]

6
crates/brk_cli/Cargo.toml
View File

@@ -10,8 +10,6 @@ repository.workspace = true
[dependencies]
bitcoincore-rpc = { workspace = true }
brk_computer = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
@@ -20,11 +18,11 @@ brk_server = { workspace = true }
brk_vecs = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }
color-eyre = "0.6.5"
log = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.3"
toml = "0.9.4"
[[bin]]
name = "brk"

6
crates/brk_cli/README.md
View File

@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A command line interface to run a Bitcoin Research Kit instance.

49
crates/brk_cli/src/config.rs
View File

@@ -4,14 +4,16 @@ use std::{
};
use bitcoincore_rpc::{self, Auth, Client};
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
use brk_fetcher::Fetcher;
use brk_server::Website;
use brk_vecs::{Computation, Format};
use clap::Parser;
use clap_derive::Parser;
use color_eyre::eyre::eyre;
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize};
use crate::{default_bitcoin_path, default_brk_path, dot_brk_path};
const DOWNLOADS: &str = "downloads";
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
#[command(version, about)]
@@ -31,16 +33,6 @@ pub struct Config {
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
computation: Option<Computation>,
/// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
format: Option<Format>,
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short = 'F', long, value_name = "BOOL")]
@@ -122,18 +114,10 @@ impl Config {
config_saved.brkdir = Some(brkdir);
}
if let Some(computation) = config_args.computation.take() {
config_saved.computation = Some(computation);
}
if let Some(fetch) = config_args.fetch.take() {
config_saved.fetch = Some(fetch);
}
if let Some(format) = config_args.format.take() {
config_saved.format = Some(format);
}
if let Some(website) = config_args.website.take() {
config_saved.website = Some(website);
}
@@ -291,6 +275,10 @@ impl Config {
self.brkdir().join("hars")
}
pub fn downloads_dir(&self) -> PathBuf {
dot_brk_path().join(DOWNLOADS)
}
fn path_cookiefile(&self) -> PathBuf {
self.rpccookiefile.as_ref().map_or_else(
|| self.bitcoindir().join(".cookie"),
@@ -329,14 +317,6 @@ impl Config {
.then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap())
}
pub fn computation(&self) -> Computation {
self.computation.unwrap_or_default()
}
pub fn format(&self) -> Format {
self.format.unwrap_or_default()
}
pub fn check_collisions(&self) -> bool {
self.check_collisions.is_some_and(|b| b)
}
@@ -349,3 +329,14 @@ impl Config {
self.watch.is_some_and(|b| b)
}
}
fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: Deserialize<'de> + Default,
{
match T::deserialize(deserializer) {
Ok(v) => Ok(v),
Err(_) => Ok(T::default()),
}
}

4
crates/brk_cli/src/lib.rs
View File

@@ -2,11 +2,11 @@
use std::{fs, thread};
use brk_core::{dot_brk_log_path, dot_brk_path};
mod config;
mod paths;
mod run;
pub use paths::*;
use run::*;
pub fn main() -> color_eyre::Result<()> {

0
crates/brk_core/src/utils/paths.rs → crates/brk_cli/src/paths.rs
View File

26
crates/brk_cli/src/run.rs
View File

@@ -2,9 +2,9 @@ use std::{thread::sleep, time::Duration};
use bitcoincore_rpc::{self, RpcApi};
use brk_computer::Computer;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_server::Server;
use brk_vecs::Exit;
use log::info;
use crate::config::Config;
@@ -13,10 +13,11 @@ pub fn run() -> color_eyre::Result<()> {
let config = Config::import()?;
let rpc = config.rpc()?;
let exit = Exit::new();
let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc);
let format = config.format();
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc);
let mut indexer = Indexer::forced_import(&config.brkdir())?;
@@ -36,13 +37,7 @@ pub fn run() -> color_eyre::Result<()> {
Ok(())
};
let mut computer = Computer::forced_import(
&config.brkdir(),
&indexer,
config.computation(),
config.fetcher(),
format,
)?;
let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?;
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -51,7 +46,12 @@ pub fn run() -> color_eyre::Result<()> {
let served_indexer = indexer.clone();
let served_computer = computer.clone();
let server = Server::new(served_indexer, served_computer, config.website())?;
let server = Server::new(
served_indexer,
served_computer,
config.website(),
&config.downloads_dir(),
)?;
let watch = config.watch();
let mcp = config.mcp();
@@ -72,7 +72,7 @@ pub fn run() -> color_eyre::Result<()> {
let starting_indexes =
indexer.index(&parser, rpc, &exit, config.check_collisions())?;
computer.compute(&mut indexer, starting_indexes, &exit)?;
computer.compute(&indexer, starting_indexes, &exit)?;
if let Some(delay) = config.delay() {
sleep(Duration::from_secs(delay))

5
crates/brk_computer/Cargo.toml
View File

@@ -10,14 +10,13 @@ repository.workspace = true
[dependencies]
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_structs = { workspace = true }
brk_error = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_vecs = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }

6
crates/brk_computer/README.md
View File

@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A dataset computer, built on top of `brk_indexer` and `brk_fetcher`. It computes any dataset you can think of and if it doesn't feel free to create an issue.

31
crates/brk_computer/examples/main.rs
View File

@@ -1,18 +1,16 @@
use std::{path::Path, thread};
use brk_computer::Computer;
use brk_exit::Exit;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vecs::{Computation, Format};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
use brk_vecs::Exit;
pub fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));
// let bitcoin_dir = brk_core::default_bitcoin_path();
// let bitcoin_dir = brk_structs::default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
@@ -20,38 +18,27 @@ pub fn main() -> color_eyre::Result<()> {
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
exit.set_ctrlc_handler();
// Can't increase main thread's stack programatically, thus we need to use another thread
thread::Builder::new()
.stack_size(256 * 1024 * 1024)
.spawn(move || -> color_eyre::Result<()> {
let parser = Parser::new(
bitcoin_dir.join("blocks"),
brk_core::default_brk_path(),
rpc,
);
.spawn(move || -> Result<()> {
let parser = Parser::new(bitcoin_dir.join("blocks"), Path::new("").to_path_buf(), rpc);
let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs");
let outputs_dir = _outputs_dir.as_path();
// let outputs_dir = Path::new("../../_outputs");
let format = Format::Raw;
let mut indexer = Indexer::forced_import(outputs_dir)?;
let fetcher = Fetcher::import(None)?;
let mut computer = Computer::forced_import(
outputs_dir,
&indexer,
Computation::Lazy,
Some(fetcher),
format,
)?;
let mut computer = Computer::forced_import(outputs_dir, &indexer, Some(fetcher))?;
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
computer.compute(&mut indexer, starting_indexes, &exit)?;
computer.compute(&indexer, starting_indexes, &exit)?;
Ok(())
})?

51
crates/brk_computer/src/all.rs
View File

@@ -1,13 +1,13 @@
use std::{path::Path, sync::Arc};
use brk_core::Version;
use brk_exit::Exit;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, Computation, File, Format};
use brk_structs::Version;
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format};
use log::info;
use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions};
use crate::{blocks, cointime, constants, fetched, indexes, market, mining, price, transactions};
use super::stateful;
@@ -20,6 +20,7 @@ pub struct Vecs {
pub blocks: blocks::Vecs,
pub mining: mining::Vecs,
pub market: market::Vecs,
pub price: Option<price::Vecs>,
pub transactions: transactions::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
@@ -32,12 +33,12 @@ impl Vecs {
file: &Arc<File>,
version: Version,
indexer: &Indexer,
fetch: bool,
fetcher: Option<Fetcher>,
computation: Computation,
format: Format,
fetched_file: &Arc<File>,
states_path: &Path,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let indexes = indexes::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
@@ -46,10 +47,19 @@ impl Vecs {
format,
)?;
let fetched = fetch.then(|| {
let fetched = fetcher.map(|fetcher| {
fetched::Vecs::forced_import(
file,
fetched_file,
fetcher,
version + VERSION + Version::ZERO,
)
.unwrap()
});
let price = fetched.is_some().then(|| {
price::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
@@ -93,7 +103,7 @@ impl Vecs {
computation,
format,
&indexes,
fetched.as_ref(),
price.as_ref(),
states_path,
)?,
transactions: transactions::Vecs::forced_import(
@@ -103,7 +113,7 @@ impl Vecs {
&indexes,
computation,
format,
fetched.as_ref(),
price.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
file,
@@ -111,10 +121,11 @@ impl Vecs {
computation,
format,
&indexes,
fetched.as_ref(),
price.as_ref(),
)?,
indexes,
fetched,
price,
})
}
@@ -122,9 +133,8 @@ impl Vecs {
&mut self,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
fetcher: Option<&mut Fetcher>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
@@ -142,11 +152,13 @@ impl Vecs {
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
self.price.as_mut().unwrap().compute(
indexer,
&self.indexes,
&starting_indexes,
fetcher.unwrap(),
fetched,
exit,
)?;
}
@@ -156,16 +168,16 @@ impl Vecs {
indexer,
&self.indexes,
&starting_indexes,
self.fetched.as_ref(),
self.price.as_ref(),
exit,
)?;
if let Some(fetched) = self.fetched.as_ref() {
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market.compute(
indexer,
&self.indexes,
fetched,
price,
&mut self.transactions,
&starting_indexes,
exit,
@@ -177,7 +189,7 @@ impl Vecs {
indexer,
&self.indexes,
&self.transactions,
self.fetched.as_ref(),
self.price.as_ref(),
&self.market,
&mut starting_indexes,
exit,
@@ -187,7 +199,7 @@ impl Vecs {
indexer,
&self.indexes,
&starting_indexes,
self.fetched.as_ref(),
self.price.as_ref(),
&self.transactions,
&self.stateful,
exit,
@@ -207,6 +219,7 @@ impl Vecs {
self.stateful.vecs(),
self.cointime.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
self.price.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()

56
crates/brk_computer/src/blocks.rs
View File

@@ -1,12 +1,12 @@
use std::sync::Arc;
use std::{path::Path, sync::Arc};
use brk_core::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
Timestamp, Version, Weight,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, File, Format};
use brk_structs::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version,
Weight,
};
use brk_vecs::{AnyCollectableVec, Computation, EagerVec, Exit, File, Format, VecIterator};
use crate::grouped::Source;
@@ -20,6 +20,8 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
pub height_to_interval: EagerVec<Height, Timestamp>,
pub height_to_vbytes: EagerVec<Height, StoredU64>,
pub difficultyepoch_to_timestamp: EagerVec<DifficultyEpoch, Timestamp>,
@@ -27,28 +29,30 @@ pub struct Vecs {
pub timeindexes_to_timestamp: ComputedVecsFromDateIndex<Timestamp>,
pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
pub indexes_to_block_size: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_block_size: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_block_vbytes: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_block_weight: ComputedVecsFromHeight<Weight>,
}
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("blocks"))?);
Ok(Self {
height_to_interval: EagerVec::forced_import(
file,
&file,
"interval",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
file,
&file,
"timestamp",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -58,7 +62,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
file,
&file,
"block_interval",
Source::None,
version + VERSION + Version::ZERO,
@@ -71,7 +75,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_block_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"block_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -81,7 +85,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
file,
&file,
"block_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -91,7 +95,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
file,
&file,
"block_size",
Source::None,
version + VERSION + Version::ZERO,
@@ -101,13 +105,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
file,
&file,
"vbytes",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
file,
&file,
"block_vbytes",
Source::None,
version + VERSION + Version::ZERO,
@@ -117,17 +121,19 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
file,
&file,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_timestamp: EagerVec::forced_import(
file,
&file,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
file,
})
}
@@ -137,7 +143,7 @@ impl Vecs {
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.timeindexes_to_timestamp.compute_all(
indexer,
indexes,
@@ -149,7 +155,8 @@ impl Vecs {
&indexes.dateindex_to_date,
|(di, d, ..)| (di, Timestamp::from(d)),
exit,
)
)?;
Ok(())
},
)?;
@@ -164,7 +171,8 @@ impl Vecs {
&indexer.vecs.height_to_weight,
|h| (h, StoredU32::from(1_u32)),
exit,
)
)?;
Ok(())
},
)?;
@@ -240,6 +248,8 @@ impl Vecs {
exit,
)?;
self.file.flush()?;
self.file.punch_holes()?;
Ok(())
}

155
crates/brk_computer/src/cointime.rs
View File

@@ -1,23 +1,25 @@
use std::sync::Arc;
use std::{path::Path, sync::Arc};
use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator};
use super::{
Indexes, fetched,
Indexes,
grouped::{
ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
Source, VecBuilderOptions,
},
indexes, stateful, transactions,
indexes, price, stateful, transactions,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
pub indexes_to_coinblocks_created: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_coinblocks_stored: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_liveliness: ComputedVecsFromHeight<StoredF64>,
@@ -46,18 +48,20 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("cointime"))?);
let compute_dollars = price.is_some();
Ok(Self {
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
file,
&file,
"coinblocks_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -67,7 +71,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
file,
&file,
"coinblocks_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -77,7 +81,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
file,
&file,
"liveliness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -87,7 +91,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
file,
&file,
"vaultedness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -97,7 +101,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
file,
&file,
"activity_to_vaultedness_ratio",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -107,7 +111,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
file,
&file,
"vaulted_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -118,7 +122,7 @@ impl Vecs {
indexes,
)?,
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
file,
&file,
"active_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -129,7 +133,7 @@ impl Vecs {
indexes,
)?,
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
file,
&file,
"thermo_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -139,7 +143,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
file,
&file,
"investor_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -149,7 +153,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
file,
&file,
"vaulted_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -159,7 +163,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
file,
&file,
"active_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -169,7 +173,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
file,
&file,
"vaulted_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -179,7 +183,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
file,
&file,
"vaulted_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -189,7 +193,7 @@ impl Vecs {
true,
)?,
indexes_to_active_price: ComputedVecsFromHeight::forced_import(
file,
&file,
"active_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -199,7 +203,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
file,
&file,
"active_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -209,7 +213,7 @@ impl Vecs {
true,
)?,
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
file,
&file,
"true_market_mean",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -219,7 +223,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
file,
&file,
"true_market_mean",
Source::None,
version + VERSION + Version::ZERO,
@@ -229,7 +233,7 @@ impl Vecs {
true,
)?,
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
file,
&file,
"cointime_value_destroyed",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -239,7 +243,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
file,
&file,
"cointime_value_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -249,7 +253,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
file,
&file,
"cointime_value_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -259,7 +263,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
file,
&file,
"cointime_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -269,7 +273,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
file,
&file,
"cointime_cap",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -279,7 +283,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
file,
&file,
"cointime_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -288,6 +292,8 @@ impl Vecs {
indexes,
true,
)?,
file,
})
}
@@ -297,11 +303,11 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
transactions: &transactions::Vecs,
stateful: &stateful::Vecs,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply;
self.indexes_to_coinblocks_created.compute_all(
@@ -315,7 +321,8 @@ impl Vecs {
circulating_supply,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)
)?;
Ok(())
},
)?;
@@ -341,7 +348,8 @@ impl Vecs {
(i, created.checked_sub(destroyed).unwrap())
},
exit,
)
)?;
Ok(())
},
)?;
@@ -360,7 +368,8 @@ impl Vecs {
.height_extra
.unwrap_cumulative(),
exit,
)
)?;
Ok(())
},
)?;
let liveliness = &self.indexes_to_liveliness;
@@ -376,7 +385,8 @@ impl Vecs {
liveliness.height.as_ref().unwrap(),
|(i, v, ..)| (i, StoredF64::from(1.0).checked_sub(v).unwrap()),
exit,
)
)?;
Ok(())
},
)?;
let vaultedness = &self.indexes_to_vaultedness;
@@ -392,14 +402,15 @@ impl Vecs {
liveliness.height.as_ref().unwrap(),
vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_vaulted_supply.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -408,14 +419,15 @@ impl Vecs {
circulating_supply,
vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_active_supply.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -424,11 +436,12 @@ impl Vecs {
circulating_supply,
liveliness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
if let Some(fetched) = fetched {
if let Some(price) = price {
let realized_cap = stateful
.utxo_cohorts
.all
@@ -465,7 +478,8 @@ impl Vecs {
.unwrap_cumulative(),
|(i, v, ..)| (i, v),
exit,
)
)?;
Ok(())
},
)?;
@@ -480,7 +494,8 @@ impl Vecs {
realized_cap,
self.indexes_to_thermo_cap.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -495,7 +510,8 @@ impl Vecs {
realized_cap,
self.indexes_to_vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -510,7 +526,8 @@ impl Vecs {
realized_cap,
self.indexes_to_liveliness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -525,14 +542,15 @@ impl Vecs {
realized_price,
self.indexes_to_vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_vaulted_price_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_vaulted_price.dateindex.unwrap_last()),
@@ -549,14 +567,15 @@ impl Vecs {
realized_price,
self.indexes_to_liveliness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_active_price_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_active_price.dateindex.unwrap_last()),
@@ -577,14 +596,15 @@ impl Vecs {
.as_ref()
.unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_true_market_mean_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_true_market_mean.dateindex.unwrap_last()),
@@ -600,10 +620,11 @@ impl Vecs {
// The price taken won't be correct for time based indexes
vec.compute_multiply(
starting_indexes.height,
&fetched.chainindexes_to_close.height,
&price.chainindexes_to_close.height,
indexes_to_coinblocks_destroyed.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -615,10 +636,11 @@ impl Vecs {
|vec, _, _, starting_indexes, exit| {
vec.compute_multiply(
starting_indexes.height,
&fetched.chainindexes_to_close.height,
&price.chainindexes_to_close.height,
self.indexes_to_coinblocks_created.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -630,10 +652,11 @@ impl Vecs {
|vec, _, _, starting_indexes, exit| {
vec.compute_multiply(
starting_indexes.height,
&fetched.chainindexes_to_close.height,
&price.chainindexes_to_close.height,
self.indexes_to_coinblocks_stored.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -652,7 +675,8 @@ impl Vecs {
.height_extra
.unwrap_cumulative(),
exit,
)
)?;
Ok(())
},
)?;
@@ -667,20 +691,23 @@ impl Vecs {
self.indexes_to_cointime_price.height.as_ref().unwrap(),
circulating_supply,
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_cointime_price_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_cointime_price.dateindex.unwrap_last()),
)?;
}
self.file.flush()?;
self.file.punch_holes()?;
Ok(())
}

58
crates/brk_computer/src/constants.rs
View File

@@ -1,9 +1,9 @@
use std::sync::Arc;
use std::{path::Path, sync::Arc};
use brk_core::{StoredU8, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, AnyVec, Computation, File, Format};
use brk_structs::{StoredU16, Version};
use brk_vecs::{AnyCollectableVec, AnyVec, Computation, Exit, File, Format};
use crate::grouped::Source;
@@ -17,23 +17,27 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub constant_0: ComputedVecsFromHeight<StoredU8>,
pub constant_1: ComputedVecsFromHeight<StoredU8>,
pub constant_50: ComputedVecsFromHeight<StoredU8>,
pub constant_100: ComputedVecsFromHeight<StoredU8>,
file: Arc<File>,
pub constant_0: ComputedVecsFromHeight<StoredU16>,
pub constant_1: ComputedVecsFromHeight<StoredU16>,
pub constant_50: ComputedVecsFromHeight<StoredU16>,
pub constant_100: ComputedVecsFromHeight<StoredU16>,
}
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("constants"))?);
Ok(Self {
constant_0: ComputedVecsFromHeight::forced_import(
file,
&file,
"constant_0",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -43,7 +47,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_1: ComputedVecsFromHeight::forced_import(
file,
&file,
"constant_1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -53,7 +57,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_50: ComputedVecsFromHeight::forced_import(
file,
&file,
"constant_50",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -63,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_100: ComputedVecsFromHeight::forced_import(
file,
&file,
"constant_100",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -72,6 +76,8 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_last(),
)?,
file,
})
}
@@ -81,7 +87,7 @@ impl Vecs {
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.constant_0.compute_all(
indexer,
indexes,
@@ -92,9 +98,10 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(0)),
|i| (i, StoredU16::new(0)),
exit,
)
)?;
Ok(())
},
)?;
@@ -108,9 +115,10 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(1)),
|i| (i, StoredU16::new(1)),
exit,
)
)?;
Ok(())
},
)?;
@@ -124,9 +132,10 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(50)),
|i| (i, StoredU16::new(50)),
exit,
)
)?;
Ok(())
},
)?;
@@ -140,12 +149,15 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(100)),
|i| (i, StoredU16::new(100)),
exit,
)
)?;
Ok(())
},
)?;
self.file.flush()?;
self.file.punch_holes()?;
Ok(())
}

1274
crates/brk_computer/src/fetched.rs
View File

File diff suppressed because it is too large Load Diff

15
crates/brk_computer/src/grouped/builder_computed.rs
View File

@@ -1,10 +1,11 @@
use std::sync::Arc;
use brk_core::{FromCoarserIndex, Result, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_structs::Version;
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, File, Format, StoredIndex,
AnyBoxedIterableVec, AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, Exit, File, Format, FromCoarserIndex, StoredIndex,
};
use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
@@ -44,11 +45,11 @@ where
version: Version,
format: Format,
computation: Computation,
source: Option<BoxedAnyIterableVec<S1I, T>>,
source: Option<AnyBoxedIterableVec<S1I, T>>,
source_extra: &EagerVecBuilder<S1I, T>,
len_source: BoxedAnyIterableVec<I, S2T>,
len_source: AnyBoxedIterableVec<I, S2T>,
options: ComputedVecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let only_one_active = options.is_only_one_active();
let suffix = |s: &str| format!("{name}_{s}");

64
crates/brk_computer/src/grouped/builder_eager.rs
View File

@@ -1,11 +1,11 @@
use std::sync::Arc;
use brk_core::{CheckedSub, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_error::{Error, Result};
use brk_structs::{CheckedSub, StoredU64, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, EagerVec, File, Format, StoredIndex, StoredType,
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, File, Format,
GenericStoredVec, StoredIndex, StoredRaw,
};
use color_eyre::eyre::ContextCompat;
use crate::utils::get_percentile;
@@ -44,7 +44,7 @@ where
version: Version,
format: Format,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let only_one_active = options.is_only_one_active();
let suffix = |s: &str| format!("{name}_{s}");
@@ -213,8 +213,9 @@ where
cumulative_vec.iter().unwrap_get_inner(index)
});
source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
cumulative = cumulative.clone() + v.into_owned();
cumulative_vec.forced_push_at(i, cumulative.clone(), exit)
cumulative += v.into_owned();
cumulative_vec.forced_push_at(i, cumulative, exit)?;
Ok(())
})?;
self.safe_flush(exit)?;
@@ -227,11 +228,11 @@ where
max_from: I,
source: &impl AnyIterableVec<I2, T>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
count_indexes: &impl AnyIterableVec<I, StoredU64>,
exit: &Exit,
) -> Result<()>
where
I2: StoredIndex + StoredType + CheckedSub<I2>,
I2: StoredIndex + StoredRaw + CheckedSub<I2>,
{
self.validate_computed_version_or_reset_file(
source.version() + first_indexes.version() + count_indexes.version(),
@@ -265,7 +266,7 @@ where
}
if let Some(last) = self.last.as_mut() {
let count_index = *count_index;
let count_index = *count_index as usize;
if count_index == 0 {
panic!("should compute last if count can be 0")
}
@@ -295,7 +296,7 @@ where
if needs_values {
source_iter.set(first_index);
let mut values = (&mut source_iter)
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
@@ -305,9 +306,9 @@ where
if let Some(max) = self.max.as_mut() {
max.forced_push_at(
i,
values
*values
.last()
.context("expect some")
.ok_or(Error::Str("expect some"))
.inspect_err(|_| {
dbg!(
&values,
@@ -320,8 +321,7 @@ where
source.name()
);
})
.unwrap()
.clone(),
.unwrap(),
exit,
)?;
}
@@ -347,7 +347,7 @@ where
}
if let Some(min) = self.min.as_mut() {
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
min.forced_push_at(i, *values.first().unwrap(), exit)?;
}
}
@@ -356,18 +356,18 @@ where
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(average) = self.average.as_mut() {
let avg = sum.clone() / len;
let avg = sum / len;
average.forced_push_at(i, avg, exit)?;
}
if needs_sum_or_cumulative {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
sum_vec.forced_push_at(i, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(i, t, exit)?;
}
}
@@ -388,11 +388,11 @@ where
max_from: I,
source: &EagerVecBuilder<I2, T>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
count_indexes: &impl AnyIterableVec<I, StoredU64>,
exit: &Exit,
) -> Result<()>
where
I2: StoredIndex + StoredType + CheckedSub<I2>,
I2: StoredIndex + StoredRaw + CheckedSub<I2>,
{
if self._90p.is_some()
|| self._75p.is_some()
@@ -440,7 +440,7 @@ where
}
if let Some(last) = self.last.as_mut() {
let count_index = *count_index;
let count_index = *count_index as usize;
if count_index == 0 {
panic!("should compute last if count can be 0")
}
@@ -464,22 +464,22 @@ where
let source_max_iter = source_max_iter.as_mut().unwrap();
source_max_iter.set(first_index);
let mut values = source_max_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
max.forced_push_at(i, *values.last().unwrap(), exit)?;
}
if let Some(min) = self.min.as_mut() {
let source_min_iter = source_min_iter.as_mut().unwrap();
source_min_iter.set(first_index);
let mut values = source_min_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
min.forced_push_at(i, *values.first().unwrap(), exit)?;
}
}
@@ -488,7 +488,7 @@ where
let source_average_iter = source_average_iter.as_mut().unwrap();
source_average_iter.set(first_index);
let values = source_average_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
@@ -504,19 +504,19 @@ where
let source_sum_iter = source_sum_iter.as_mut().unwrap();
source_sum_iter.set(first_index);
let values = source_sum_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
sum_vec.forced_push_at(i, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(i, t, exit)?;
}
}

23
crates/brk_computer/src/grouped/computed.rs Normal file
View File

@@ -0,0 +1,23 @@
use std::ops::{Add, AddAssign, Div};
use brk_vecs::StoredCompressed;
pub trait ComputedType
where
Self: StoredCompressed
+ From<usize>
+ Div<usize, Output = Self>
+ Add<Output = Self>
+ AddAssign
+ Ord,
{
}
impl<T> ComputedType for T where
T: StoredCompressed
+ From<usize>
+ Div<usize, Output = Self>
+ Add<Output = Self>
+ AddAssign
+ Ord
{
}

19
crates/brk_computer/src/grouped/from_dateindex.rs
View File

@@ -1,13 +1,14 @@
use std::sync::Arc;
use brk_core::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex,
YearIndex,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File,
Format,
};
use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
@@ -45,7 +46,7 @@ where
computation: Computation,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let dateindex = source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
});
@@ -141,7 +142,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<DateIndex, T>,
@@ -169,7 +170,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(dateindex) = dateindex {
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;

20
crates/brk_computer/src/grouped/from_height.rs
View File

@@ -1,13 +1,15 @@
use std::sync::Arc;
use brk_core::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result,
SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File,
Format,
};
use crate::{
@@ -53,7 +55,7 @@ where
computation: Computation,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let height = source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
});
@@ -164,7 +166,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(&mut EagerVec<Height, T>, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>,
{
@@ -186,7 +188,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
height_vec: Option<&impl AnyIterableVec<Height, T>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(height) = height_vec {
self.height_extra
.extend(starting_indexes.height, height, exit)?;

11
crates/brk_computer/src/grouped/from_height_strict.rs
View File

@@ -1,9 +1,10 @@
use std::sync::Arc;
use brk_core::{DifficultyEpoch, Height, Result, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, EagerVec, File, Format};
use brk_structs::{DifficultyEpoch, Height, Version};
use brk_vecs::{AnyCollectableVec, EagerVec, Exit, File, Format};
use crate::{Indexes, indexes};
@@ -33,7 +34,7 @@ where
version: Version,
format: Format,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let height =
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?;
@@ -68,7 +69,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(&mut EagerVec<Height, T>, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>,
{

26
crates/brk_computer/src/grouped/from_txindex.rs
View File

@@ -1,20 +1,20 @@
use std::sync::Arc;
use brk_core::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Result, Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec,
File, Format, StoredIndex, VecIterator,
AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec,
Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{
Indexes, fetched,
Indexes,
grouped::{ComputedVecBuilder, Source},
indexes,
indexes, price,
};
use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};
@@ -54,7 +54,7 @@ where
computation: Computation,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let txindex = source.is_compute().then(|| {
Box::new(
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
@@ -170,7 +170,7 @@ where
// starting_indexes: &Indexes,
// exit: &Exit,
// mut compute: F,
// ) -> color_eyre::Result<()>
// ) -> Result<()>
// where
// F: FnMut(
// &mut EagerVec<TxIndex, T>,
@@ -495,7 +495,7 @@ impl ComputedVecsFromTxindex<Dollars> {
exit: &Exit,
bitcoin: &ComputedVecsFromTxindex<Bitcoin>,
txindex: Option<&impl CollectableVec<TxIndex, Dollars>>,
fetched: &fetched::Vecs,
price: &price::Vecs,
) -> Result<()> {
let txindex_version = if let Some(txindex) = txindex {
txindex.version()
@@ -508,7 +508,7 @@ impl ComputedVecsFromTxindex<Dollars> {
let starting_index = self.height.starting_index(starting_indexes.height);
let mut close_iter = fetched.chainindexes_to_close.height.into_iter();
let mut close_iter = price.chainindexes_to_close.height.into_iter();
(starting_index.unwrap_to_usize()..indexer.vecs.height_to_weight.len())
.map(Height::from)

4
crates/brk_computer/src/grouped/mod.rs
View File

@@ -1,12 +1,12 @@
mod builder_computed;
mod builder_eager;
mod computed;
mod from_dateindex;
mod from_height;
mod from_height_strict;
mod from_txindex;
mod ratio_from_dateindex;
mod source;
mod r#type;
mod value_from_dateindex;
mod value_from_height;
mod value_from_txindex;
@@ -14,13 +14,13 @@ mod value_height;
pub use builder_computed::*;
pub use builder_eager::*;
use computed::*;
pub use from_dateindex::*;
pub use from_height::*;
pub use from_height_strict::*;
pub use from_txindex::*;
pub use ratio_from_dateindex::*;
pub use source::*;
use r#type::*;
pub use value_from_dateindex::*;
pub use value_from_height::*;
pub use value_from_txindex::*;

93
crates/brk_computer/src/grouped/ratio_from_dateindex.rs
View File

@@ -1,14 +1,14 @@
use std::{f32, sync::Arc};
use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Date, DateIndex, Dollars, StoredF32, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format,
StoredIndex, VecIterator,
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec,
Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{Indexes, fetched, grouped::source::Source, indexes, utils::get_percentile};
use crate::{Indexes, grouped::source::Source, indexes, price, utils::get_percentile};
use super::{ComputedVecsFromDateIndex, VecBuilderOptions};
@@ -68,7 +68,7 @@ impl ComputedRatioVecsFromDateIndex {
computation: Computation,
indexes: &indexes::Vecs,
extended: bool,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let options = VecBuilderOptions::default().add_last();
Ok(Self {
@@ -570,11 +570,11 @@ impl ComputedRatioVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: &fetched::Vecs,
price: &price::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<DateIndex, Dollars>,
@@ -596,7 +596,7 @@ impl ComputedRatioVecsFromDateIndex {
self.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
date_to_price_opt,
@@ -607,16 +607,16 @@ impl ComputedRatioVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: &fetched::Vecs,
price: &price::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
date_to_price_opt: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
let date_to_price = date_to_price_opt.unwrap_or_else(|| unsafe {
std::mem::transmute(&self.price.as_ref().unwrap().dateindex)
});
let closes = fetched.timeindexes_to_close.dateindex.as_ref().unwrap();
let closes = price.timeindexes_to_close.dateindex.as_ref().unwrap();
self.ratio.compute_all(
indexer,
@@ -637,7 +637,8 @@ impl ComputedRatioVecsFromDateIndex {
}
},
exit,
)
)?;
Ok(())
},
)?;
@@ -659,7 +660,8 @@ impl ComputedRatioVecsFromDateIndex {
usize::MAX,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -675,7 +677,8 @@ impl ComputedRatioVecsFromDateIndex {
7,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -691,7 +694,8 @@ impl ComputedRatioVecsFromDateIndex {
30,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -707,7 +711,8 @@ impl ComputedRatioVecsFromDateIndex {
365,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -723,7 +728,8 @@ impl ComputedRatioVecsFromDateIndex {
4 * 365,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -756,7 +762,8 @@ impl ComputedRatioVecsFromDateIndex {
)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -766,7 +773,8 @@ impl ComputedRatioVecsFromDateIndex {
.try_for_each(|v| -> Result<()> {
v.validate_computed_version_or_reset_file(
Version::ZERO + v.inner_version() + ratio_version,
)
)?;
Ok(())
})?;
let starting_dateindex = self
@@ -1194,7 +1202,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1220,7 +1229,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1246,7 +1256,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1272,7 +1283,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1298,7 +1310,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1324,7 +1337,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1350,7 +1364,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1376,7 +1391,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1402,7 +1418,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1428,7 +1445,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1454,7 +1472,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1480,7 +1499,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1496,7 +1516,8 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_sma.as_ref().unwrap().dateindex.as_ref().unwrap(),
self.ratio_sd.as_ref().unwrap().dateindex.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -1522,7 +1543,8 @@ impl ComputedRatioVecsFromDateIndex {
.as_ref()
.unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -1548,7 +1570,8 @@ impl ComputedRatioVecsFromDateIndex {
.as_ref()
.unwrap(),
exit,
)
)?;
Ok(())
},
)?;

14
crates/brk_computer/src/grouped/source.rs
View File

@@ -1,10 +1,10 @@
use brk_vecs::BoxedAnyIterableVec;
use brk_vecs::AnyBoxedIterableVec;
#[derive(Clone)]
pub enum Source<I, T> {
Compute,
None,
Vec(BoxedAnyIterableVec<I, T>),
Vec(AnyBoxedIterableVec<I, T>),
}
impl<I, T> Source<I, T> {
@@ -20,7 +20,7 @@ impl<I, T> Source<I, T> {
matches!(self, Self::Vec(_))
}
pub fn vec(self) -> Option<BoxedAnyIterableVec<I, T>> {
pub fn vec(self) -> Option<AnyBoxedIterableVec<I, T>> {
match self {
Self::Vec(v) => Some(v),
_ => None,
@@ -34,14 +34,14 @@ impl<I, T> From<bool> for Source<I, T> {
}
}
impl<I, T> From<BoxedAnyIterableVec<I, T>> for Source<I, T> {
fn from(value: BoxedAnyIterableVec<I, T>) -> Self {
impl<I, T> From<AnyBoxedIterableVec<I, T>> for Source<I, T> {
fn from(value: AnyBoxedIterableVec<I, T>) -> Self {
Self::Vec(value)
}
}
impl<I, T> From<Option<BoxedAnyIterableVec<I, T>>> for Source<I, T> {
fn from(value: Option<BoxedAnyIterableVec<I, T>>) -> Self {
impl<I, T> From<Option<AnyBoxedIterableVec<I, T>>> for Source<I, T> {
fn from(value: Option<AnyBoxedIterableVec<I, T>>) -> Self {
if let Some(v) = value {
Self::Vec(v)
} else {

14
crates/brk_computer/src/grouped/type.rs
View File

@@ -1,14 +0,0 @@
use std::ops::{Add, AddAssign, Div};
use brk_vecs::StoredType;
pub trait ComputedType
where
Self:
StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self> + AddAssign + Ord,
{
}
impl<T> ComputedType for T where
T: StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self> + AddAssign + Ord
{
}

29
crates/brk_computer/src/grouped/value_from_dateindex.rs
View File

@@ -1,11 +1,18 @@
use std::sync::Arc;
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
use brk_structs::{Bitcoin, DateIndex, Dollars, Sats, Version};
use brk_vecs::{
AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec,
};
use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
use crate::{
Indexes,
grouped::ComputedVecsFromDateIndex,
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
};
use super::{Source, VecBuilderOptions};
@@ -30,7 +37,7 @@ impl ComputedValueVecsFromDateIndex {
options: VecBuilderOptions,
compute_dollars: bool,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import(
file,
@@ -72,11 +79,11 @@ impl ComputedValueVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<DateIndex, Sats>,
@@ -95,7 +102,7 @@ impl ComputedValueVecsFromDateIndex {
)?;
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, dateindex)?;
self.compute_rest(indexer, indexes, price, starting_indexes, exit, dateindex)?;
Ok(())
}
@@ -104,11 +111,11 @@ impl ComputedValueVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl CollectableVec<DateIndex, Sats>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(dateindex) = dateindex {
self.sats
.compute_rest(indexes, starting_indexes, exit, Some(dateindex))?;
@@ -144,7 +151,7 @@ impl ComputedValueVecsFromDateIndex {
}
let dateindex_to_bitcoin = self.bitcoin.dateindex.as_ref().unwrap();
let dateindex_to_close = fetched
let dateindex_to_close = price
.as_ref()
.unwrap()
.timeindexes_to_close

29
crates/brk_computer/src/grouped/value_from_height.rs
View File

@@ -1,11 +1,18 @@
use std::sync::Arc;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_vecs::{
AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec,
};
use crate::{Indexes, fetched, grouped::Source, indexes};
use crate::{
Indexes,
grouped::Source,
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
};
use super::{ComputedVecsFromHeight, VecBuilderOptions};
@@ -30,7 +37,7 @@ impl ComputedValueVecsFromHeight {
options: VecBuilderOptions,
compute_dollars: bool,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromHeight::forced_import(
file,
@@ -72,11 +79,11 @@ impl ComputedValueVecsFromHeight {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<Height, Sats>,
@@ -95,7 +102,7 @@ impl ComputedValueVecsFromHeight {
)?;
let height: Option<&StoredVec<Height, Sats>> = None;
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, height)?;
self.compute_rest(indexer, indexes, price, starting_indexes, exit, height)?;
Ok(())
}
@@ -104,11 +111,11 @@ impl ComputedValueVecsFromHeight {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&impl CollectableVec<Height, Sats>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(height) = height {
self.sats
.compute_rest(indexes, starting_indexes, exit, Some(height))?;
@@ -144,7 +151,7 @@ impl ComputedValueVecsFromHeight {
}
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
let height_to_close = &price.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_all(

30
crates/brk_computer/src/grouped/value_from_txindex.rs
View File

@@ -1,14 +1,14 @@
use std::sync::Arc;
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_vecs::{
AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3,
File, Format, LazyVecFrom1, StoredIndex, StoredVec,
AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Computation, ComputedVecFrom3,
Exit, File, Format, LazyVecFrom1, StoredIndex, StoredVec,
};
use crate::{Indexes, fetched, grouped::Source, indexes};
use crate::{Indexes, grouped::Source, indexes, price};
use super::{ComputedVecsFromTxindex, VecBuilderOptions};
@@ -45,10 +45,10 @@ impl ComputedValueVecsFromTxindex {
version: Version,
computation: Computation,
format: Format,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
) -> Result<Self> {
let compute_dollars = price.is_some();
let name_in_btc = format!("{name}_in_btc");
let name_in_usd = format!("{name}_in_usd");
@@ -89,7 +89,7 @@ impl ComputedValueVecsFromTxindex {
options,
)?;
let dollars_txindex = fetched.map(|fetched| {
let dollars_txindex = price.map(|price| {
ComputedVecFrom3::forced_import_or_init_from_3(
computation,
file,
@@ -98,7 +98,7 @@ impl ComputedValueVecsFromTxindex {
format,
bitcoin_txindex.boxed_clone(),
indexes.txindex_to_height.boxed_clone(),
fetched.chainindexes_to_close.height.boxed_clone(),
price.chainindexes_to_close.height.boxed_clone(),
|txindex: TxIndex,
txindex_to_btc_iter,
txindex_to_height_iter,
@@ -145,11 +145,11 @@ impl ComputedValueVecsFromTxindex {
// &mut self,
// indexer: &Indexer,
// indexes: &indexes::Vecs,
// fetched: Option<&marketprice::Vecs>,
// price: Option<&marketprice::Vecs>,
// starting_indexes: &Indexes,
// exit: &Exit,
// mut compute: F,
// ) -> color_eyre::Result<()>
// ) -> Result<()>
// where
// F: FnMut(
// &mut EagerVec<TxIndex, Sats>,
@@ -187,8 +187,8 @@ impl ComputedValueVecsFromTxindex {
starting_indexes: &Indexes,
exit: &Exit,
txindex: Option<&impl CollectableVec<TxIndex, Sats>>,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<()> {
price: Option<&price::Vecs>,
) -> Result<()> {
if let Some(txindex) = txindex {
self.sats
.compute_rest(indexer, indexes, starting_indexes, exit, Some(txindex))?;
@@ -223,7 +223,7 @@ impl ComputedValueVecsFromTxindex {
exit,
&self.bitcoin,
Some(dollars_txindex),
fetched.as_ref().unwrap(),
price.as_ref().unwrap(),
)?;
}

27
crates/brk_computer/src/grouped/value_height.rs
View File

@@ -1,11 +1,16 @@
use std::sync::Arc;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, File, Format, StoredVec};
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, Exit, File, Format, StoredVec};
use crate::{Indexes, fetched, grouped::Source, indexes};
use crate::{
Indexes,
grouped::Source,
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
};
#[derive(Clone)]
pub struct ComputedHeightValueVecs {
@@ -24,7 +29,7 @@ impl ComputedHeightValueVecs {
version: Version,
format: Format,
compute_dollars: bool,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self {
sats: source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
@@ -52,11 +57,11 @@ impl ComputedHeightValueVecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<Height, Sats>,
@@ -75,18 +80,18 @@ impl ComputedHeightValueVecs {
)?;
let height: Option<&StoredVec<Height, Sats>> = None;
self.compute_rest(fetched, starting_indexes, exit, height)?;
self.compute_rest(price, starting_indexes, exit, height)?;
Ok(())
}
pub fn compute_rest(
&mut self,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&impl CollectableVec<Height, Sats>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(height) = height {
self.bitcoin
.compute_from_sats(starting_indexes.height, height, exit)?;
@@ -99,7 +104,7 @@ impl ComputedHeightValueVecs {
}
let height_to_bitcoin = &self.bitcoin;
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
let height_to_close = &price.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_from_bitcoin(

175
crates/brk_computer/src/indexes.rs
View File

@@ -1,36 +1,38 @@
use std::{ops::Deref, sync::Arc};
use std::{ops::Deref, path::Path, sync::Arc};
use brk_core::{
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2AAddressIndex, P2ABytes, P2MSOutputIndex,
P2PK33AddressIndex, P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex,
P2PKHBytes, P2SHAddressIndex, P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex,
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredUsize,
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64,
Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vecs::{
AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, File, Format, StoredIndex, VecIterator,
AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, Exit, File, Format, StoredIndex, VecIterator,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
pub dateindex_to_date: EagerVec<DateIndex, Date>,
pub dateindex_to_dateindex: EagerVec<DateIndex, DateIndex>,
pub dateindex_to_first_height: EagerVec<DateIndex, Height>,
pub dateindex_to_height_count: EagerVec<DateIndex, StoredUsize>,
pub dateindex_to_height_count: EagerVec<DateIndex, StoredU64>,
pub dateindex_to_monthindex: EagerVec<DateIndex, MonthIndex>,
pub dateindex_to_weekindex: EagerVec<DateIndex, WeekIndex>,
pub decadeindex_to_decadeindex: EagerVec<DecadeIndex, DecadeIndex>,
pub decadeindex_to_first_yearindex: EagerVec<DecadeIndex, YearIndex>,
pub decadeindex_to_yearindex_count: EagerVec<DecadeIndex, StoredUsize>,
pub decadeindex_to_yearindex_count: EagerVec<DecadeIndex, StoredU64>,
pub difficultyepoch_to_difficultyepoch: EagerVec<DifficultyEpoch, DifficultyEpoch>,
pub difficultyepoch_to_first_height: EagerVec<DifficultyEpoch, Height>,
pub difficultyepoch_to_height_count: EagerVec<DifficultyEpoch, StoredUsize>,
pub difficultyepoch_to_height_count: EagerVec<DifficultyEpoch, StoredU64>,
pub emptyoutputindex_to_emptyoutputindex:
ComputedVecFrom1<EmptyOutputIndex, EmptyOutputIndex, EmptyOutputIndex, TxIndex>,
pub halvingepoch_to_first_height: EagerVec<HalvingEpoch, Height>,
@@ -42,9 +44,9 @@ pub struct Vecs {
pub height_to_halvingepoch: EagerVec<Height, HalvingEpoch>,
pub height_to_height: EagerVec<Height, Height>,
pub height_to_timestamp_fixed: EagerVec<Height, Timestamp>,
pub height_to_txindex_count: EagerVec<Height, StoredUsize>,
pub height_to_txindex_count: EagerVec<Height, StoredU64>,
pub inputindex_to_inputindex: ComputedVecFrom1<InputIndex, InputIndex, InputIndex, OutputIndex>,
pub monthindex_to_dateindex_count: EagerVec<MonthIndex, StoredUsize>,
pub monthindex_to_dateindex_count: EagerVec<MonthIndex, StoredU64>,
pub monthindex_to_first_dateindex: EagerVec<MonthIndex, DateIndex>,
pub monthindex_to_monthindex: EagerVec<MonthIndex, MonthIndex>,
pub monthindex_to_quarterindex: EagerVec<MonthIndex, QuarterIndex>,
@@ -73,39 +75,41 @@ pub struct Vecs {
pub p2wshaddressindex_to_p2wshaddressindex:
ComputedVecFrom1<P2WSHAddressIndex, P2WSHAddressIndex, P2WSHAddressIndex, P2WSHBytes>,
pub quarterindex_to_first_monthindex: EagerVec<QuarterIndex, MonthIndex>,
pub quarterindex_to_monthindex_count: EagerVec<QuarterIndex, StoredUsize>,
pub quarterindex_to_monthindex_count: EagerVec<QuarterIndex, StoredU64>,
pub quarterindex_to_quarterindex: EagerVec<QuarterIndex, QuarterIndex>,
pub semesterindex_to_first_monthindex: EagerVec<SemesterIndex, MonthIndex>,
pub semesterindex_to_monthindex_count: EagerVec<SemesterIndex, StoredUsize>,
pub semesterindex_to_monthindex_count: EagerVec<SemesterIndex, StoredU64>,
pub semesterindex_to_semesterindex: EagerVec<SemesterIndex, SemesterIndex>,
pub txindex_to_height: EagerVec<TxIndex, Height>,
pub txindex_to_input_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, InputIndex, InputIndex, OutputIndex>,
ComputedVecFrom2<TxIndex, StoredU64, TxIndex, InputIndex, InputIndex, OutputIndex>,
pub txindex_to_output_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, OutputIndex, OutputIndex, Sats>,
ComputedVecFrom2<TxIndex, StoredU64, TxIndex, OutputIndex, OutputIndex, Sats>,
pub txindex_to_txindex: ComputedVecFrom1<TxIndex, TxIndex, TxIndex, Txid>,
pub unknownoutputindex_to_unknownoutputindex:
ComputedVecFrom1<UnknownOutputIndex, UnknownOutputIndex, UnknownOutputIndex, TxIndex>,
pub weekindex_to_dateindex_count: EagerVec<WeekIndex, StoredUsize>,
pub weekindex_to_dateindex_count: EagerVec<WeekIndex, StoredU64>,
pub weekindex_to_first_dateindex: EagerVec<WeekIndex, DateIndex>,
pub weekindex_to_weekindex: EagerVec<WeekIndex, WeekIndex>,
pub yearindex_to_decadeindex: EagerVec<YearIndex, DecadeIndex>,
pub yearindex_to_first_monthindex: EagerVec<YearIndex, MonthIndex>,
pub yearindex_to_monthindex_count: EagerVec<YearIndex, StoredUsize>,
pub yearindex_to_monthindex_count: EagerVec<YearIndex, StoredU64>,
pub yearindex_to_yearindex: EagerVec<YearIndex, YearIndex>,
}
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
parent: &Path,
version: Version,
indexer: &Indexer,
computation: Computation,
format: Format,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("indexes"))?);
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"outputindex",
version + VERSION + Version::ZERO,
format,
@@ -115,7 +119,7 @@ impl Vecs {
let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"inputindex",
version + VERSION + Version::ZERO,
format,
@@ -125,7 +129,7 @@ impl Vecs {
let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"txindex",
version + VERSION + Version::ZERO,
format,
@@ -135,7 +139,7 @@ impl Vecs {
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation,
file,
&file,
"input_count",
version + VERSION + Version::ZERO,
format,
@@ -151,14 +155,14 @@ impl Vecs {
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_owned()))
.unwrap_or_else(|| inputindex_to_outputindex_iter.len());
StoredUsize::from((start..end).count())
StoredU64::from((start..end).count())
})
},
)?;
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation,
file,
&file,
"output_count",
version + VERSION + Version::ZERO,
format,
@@ -174,14 +178,14 @@ impl Vecs {
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_owned()))
.unwrap_or_else(|| outputindex_to_value_iter.len());
StoredUsize::from((start..end).count())
StoredU64::from((start..end).count())
})
},
)?;
let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2pk33addressindex",
version + VERSION + Version::ZERO,
format,
@@ -190,7 +194,7 @@ impl Vecs {
)?;
let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2pk65addressindex",
version + VERSION + Version::ZERO,
format,
@@ -199,7 +203,7 @@ impl Vecs {
)?;
let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2pkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -208,7 +212,7 @@ impl Vecs {
)?;
let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2shaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -217,7 +221,7 @@ impl Vecs {
)?;
let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2traddressindex",
version + VERSION + Version::ZERO,
format,
@@ -226,7 +230,7 @@ impl Vecs {
)?;
let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2wpkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -235,7 +239,7 @@ impl Vecs {
)?;
let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2wshaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -244,7 +248,7 @@ impl Vecs {
)?;
let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2aaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -253,7 +257,7 @@ impl Vecs {
)?;
let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"p2msoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -262,7 +266,7 @@ impl Vecs {
)?;
let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"emptyoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -271,7 +275,7 @@ impl Vecs {
)?;
let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"unknownoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -280,7 +284,7 @@ impl Vecs {
)?;
let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"opreturnindex",
version + VERSION + Version::ZERO,
format,
@@ -308,263 +312,265 @@ impl Vecs {
unknownoutputindex_to_unknownoutputindex,
dateindex_to_date: EagerVec::forced_import(
file,
&file,
"date",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
file,
&file,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_first_height: EagerVec::forced_import(
file,
&file,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_monthindex: EagerVec::forced_import(
file,
&file,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_weekindex: EagerVec::forced_import(
file,
&file,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_decadeindex: EagerVec::forced_import(
file,
&file,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_first_yearindex: EagerVec::forced_import(
file,
&file,
"first_yearindex",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
file,
&file,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_first_height: EagerVec::forced_import(
file,
&file,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_first_height: EagerVec::forced_import(
file,
&file,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_halvingepoch: EagerVec::forced_import(
file,
&file,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date: EagerVec::forced_import(
file,
&file,
"date",
version + VERSION + Version::ZERO,
format,
)?,
height_to_difficultyepoch: EagerVec::forced_import(
file,
&file,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_halvingepoch: EagerVec::forced_import(
file,
&file,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_height: EagerVec::forced_import(
file,
&file,
"height",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_first_dateindex: EagerVec::forced_import(
file,
&file,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_monthindex: EagerVec::forced_import(
file,
&file,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_quarterindex: EagerVec::forced_import(
file,
&file,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_semesterindex: EagerVec::forced_import(
file,
&file,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_yearindex: EagerVec::forced_import(
file,
&file,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_first_monthindex: EagerVec::forced_import(
file,
&file,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_first_monthindex: EagerVec::forced_import(
file,
&file,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_first_dateindex: EagerVec::forced_import(
file,
&file,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_first_monthindex: EagerVec::forced_import(
file,
&file,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_quarterindex: EagerVec::forced_import(
file,
&file,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_semesterindex: EagerVec::forced_import(
file,
&file,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_weekindex: EagerVec::forced_import(
file,
&file,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_decadeindex: EagerVec::forced_import(
file,
&file,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_yearindex: EagerVec::forced_import(
file,
&file,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date_fixed: EagerVec::forced_import(
file,
&file,
"date_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_dateindex: EagerVec::forced_import(
file,
&file,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_height: EagerVec::forced_import(
file,
&file,
"height",
version + VERSION + Version::ZERO,
format,
)?,
height_to_timestamp_fixed: EagerVec::forced_import(
file,
&file,
"timestamp_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_txindex_count: EagerVec::forced_import(
file,
&file,
"txindex_count",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_height_count: EagerVec::forced_import(
file,
&file,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_dateindex_count: EagerVec::forced_import(
file,
&file,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_height_count: EagerVec::forced_import(
file,
&file,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_dateindex_count: EagerVec::forced_import(
file,
&file,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_monthindex_count: EagerVec::forced_import(
file,
&file,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_monthindex_count: EagerVec::forced_import(
file,
&file,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_monthindex_count: EagerVec::forced_import(
file,
&file,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_yearindex_count: EagerVec::forced_import(
file,
&file,
"yearindex_count",
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_txindex: EagerVec::forced_import(
file,
&file,
"txindex",
version + VERSION + Version::ZERO,
format,
)?,
file,
})
}
@@ -573,7 +579,7 @@ impl Vecs {
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> color_eyre::Result<Indexes> {
) -> Result<Indexes> {
// ---
// OutputIndex
// ---
@@ -1110,6 +1116,9 @@ impl Vecs {
exit,
)?;
self.file.flush()?;
self.file.punch_holes()?;
Ok(Indexes {
indexes: starting_indexes,
dateindex: starting_dateindex,

223
crates/brk_computer/src/lib.rs
View File

@@ -3,16 +3,15 @@
#![doc = include_str!("../examples/main.rs")]
#![doc = "```"]
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_core::Version;
use brk_exit::Exit;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vecs::{Computation, File, Format, PAGE_SIZE};
use brk_structs::Version;
use brk_vecs::{AnyCollectableVec, Computation, Exit, Format};
use log::info;
mod all;
mod blocks;
mod cointime;
mod constants;
@@ -21,8 +20,10 @@ mod grouped;
mod indexes;
mod market;
mod mining;
mod price;
mod stateful;
mod states;
mod traits;
mod transactions;
mod utils;
@@ -32,9 +33,16 @@ use states::*;
#[derive(Clone)]
pub struct Computer {
file: Arc<File>,
fetcher: Option<Fetcher>,
pub vecs: all::Vecs,
pub indexes: indexes::Vecs,
pub constants: constants::Vecs,
pub blocks: blocks::Vecs,
pub mining: mining::Vecs,
pub market: market::Vecs,
pub price: Option<price::Vecs>,
pub transactions: transactions::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
pub cointime: cointime::Vecs,
}
const VERSION: Version = Version::ONE;
@@ -42,50 +50,193 @@ const VERSION: Version = Version::ONE;
impl Computer {
/// Do NOT import multiple times or things will break !!!
pub fn forced_import(
outputs_dir: &Path,
outputs_path: &Path,
indexer: &Indexer,
computation: Computation,
fetcher: Option<Fetcher>,
format: Format,
) -> color_eyre::Result<Self> {
let computed_path = outputs_dir.join("computed");
let states_path = computed_path.join("states");
) -> Result<Self> {
let computed_path = outputs_path.join("computed");
let file = Arc::new(File::open(&computed_path.join("vecs"))?);
file.set_min_len(PAGE_SIZE * 100_000_000)?;
file.set_min_regions(50_000)?;
let computation = Computation::Lazy;
let format = Format::Compressed;
let file_fetched = Arc::new(File::open(&outputs_dir.join("fetched/vecs"))?);
let indexes = indexes::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
indexer,
computation,
format,
)?;
Ok(Self {
vecs: all::Vecs::import(
&file,
let fetched = fetcher.map(|fetcher| {
fetched::Vecs::forced_import(outputs_path, fetcher, VERSION + Version::ZERO).unwrap()
});
let price = fetched.is_some().then(|| {
price::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
indexer,
fetcher.is_some(),
computation,
format,
&file_fetched,
&states_path,
&indexes,
)
.unwrap()
});
Ok(Self {
blocks: blocks::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
fetcher,
file,
mining: mining::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
constants: constants::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
market: market::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
stateful: stateful::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
price.as_ref(),
&computed_path.join("states"),
)?,
transactions: transactions::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
indexer,
&indexes,
computation,
format,
price.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
price.as_ref(),
)?,
indexes,
fetched,
price,
})
}
}
impl Computer {
pub fn compute(
&mut self,
indexer: &mut Indexer,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
info!("Computing...");
self.vecs
.compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)?;
self.file.flush()?;
self.file.punch_holes()?;
) -> Result<()> {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
self.constants
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing blocks...");
self.blocks
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing mining...");
self.mining
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
self.price.as_mut().unwrap().compute(
indexer,
&self.indexes,
&starting_indexes,
fetched,
exit,
)?;
}
info!("Computing transactions...");
self.transactions.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
exit,
)?;
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market.compute(
indexer,
&self.indexes,
price,
&mut self.transactions,
&starting_indexes,
exit,
)?;
}
info!("Computing stateful...");
self.stateful.compute(
indexer,
&self.indexes,
&self.transactions,
self.price.as_ref(),
&self.market,
&mut starting_indexes,
exit,
)?;
self.cointime.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
&self.transactions,
&self.stateful,
exit,
)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.constants.vecs(),
self.indexes.vecs(),
self.blocks.vecs(),
self.mining.vecs(),
self.market.vecs(),
self.transactions.vecs(),
self.stateful.vecs(),
self.cointime.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
self.price.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}

493
crates/brk_computer/src/market.rs
View File

File diff suppressed because it is too large Load Diff

34
crates/brk_computer/src/mining.rs
View File

@@ -1,9 +1,9 @@
use std::sync::Arc;
use std::{path::Path, sync::Arc};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator};
use crate::grouped::Source;
@@ -17,6 +17,8 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex<DifficultyEpoch>,
pub indexes_to_halvingepoch: ComputedVecsFromDateIndex<HalvingEpoch>,
@@ -24,15 +26,17 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("mining"))?);
Ok(Self {
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
file,
&file,
"difficulty",
Source::None,
version + VERSION + Version::ZERO,
@@ -42,7 +46,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
file,
&file,
"difficultyepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -52,7 +56,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
file,
&file,
"halvingepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -61,6 +65,8 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_last(),
)?,
file,
})
}
@@ -70,7 +76,7 @@ impl Vecs {
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
self.indexes_to_difficultyepoch.compute_all(
indexer,
@@ -91,7 +97,8 @@ impl Vecs {
)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -115,7 +122,8 @@ impl Vecs {
)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -126,6 +134,8 @@ impl Vecs {
Some(&indexer.vecs.height_to_difficulty),
)?;
self.file.flush()?;
self.file.punch_holes()?;
Ok(())
}

1273
crates/brk_computer/src/price.rs Normal file
View File

File diff suppressed because it is too large Load Diff

35
crates/brk_computer/src/stateful/address_cohort.rs
View File

@@ -1,16 +1,17 @@
use std::{ops::Deref, path::Path, sync::Arc};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, VecIterator,
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File,
Format, GenericStoredVec, VecIterator,
};
use crate::{
Indexes, fetched,
Indexes,
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, market,
indexes, market, price,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
@@ -28,8 +29,8 @@ pub struct Vecs {
pub inner: common::Vecs,
pub height_to_address_count: EagerVec<Height, StoredUsize>,
pub indexes_to_address_count: ComputedVecsFromHeight<StoredUsize>,
pub height_to_address_count: EagerVec<Height, StoredU64>,
pub indexes_to_address_count: ComputedVecsFromHeight<StoredU64>,
}
impl Vecs {
@@ -41,11 +42,11 @@ impl Vecs {
format: Format,
version: Version,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
states_path: &Path,
compute_relative_to_all: bool,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
) -> Result<Self> {
let compute_dollars = price.is_some();
let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}"));
@@ -79,7 +80,7 @@ impl Vecs {
format,
version,
indexes,
fetched,
price,
compute_relative_to_all,
false,
)?,
@@ -170,10 +171,10 @@ impl DynCohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.indexes_to_address_count.compute_rest(
indexes,
starting_indexes,
@@ -182,7 +183,7 @@ impl DynCohortVecs for Vecs {
)?;
self.inner
.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)
}
fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -223,7 +224,7 @@ impl CohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -231,11 +232,11 @@ impl CohortVecs for Vecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.inner.compute_rest_part2(
indexer,
indexes,
fetched,
price,
starting_indexes,
market,
height_to_supply,

112
crates/brk_computer/src/stateful/address_cohorts.rs
View File

@@ -1,16 +1,16 @@
use std::{path::Path, sync::Arc};
use brk_core::{
AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
Dollars, GroupFilter, Height, Result, Version,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyIterableVec, Computation, File, Format};
use brk_structs::{
AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
Dollars, GroupFilter, Height, Version,
};
use brk_vecs::{AnyIterableVec, Computation, Exit, File, Format};
use derive_deref::{Deref, DerefMut};
use crate::{
Indexes, fetched, indexes, market,
Indexes, indexes, market, price,
stateful::{
address_cohort,
r#trait::{CohortVecs, DynCohortVecs},
@@ -29,9 +29,9 @@ impl Vecs {
_computation: Computation,
format: Format,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
states_path: &Path,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self(
AddressGroups {
amount_range: ByAmountRange {
@@ -42,7 +42,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -53,7 +53,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -64,7 +64,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -75,7 +75,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -86,7 +86,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -97,7 +97,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -108,7 +108,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -119,7 +119,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -130,7 +130,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -141,7 +141,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -152,7 +152,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -163,7 +163,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -174,7 +174,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -185,7 +185,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -196,7 +196,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -209,7 +209,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -220,7 +220,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -231,7 +231,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -242,7 +242,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -253,7 +253,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -264,7 +264,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -275,7 +275,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -286,7 +286,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -297,7 +297,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -308,7 +308,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -319,7 +319,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -330,7 +330,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -341,7 +341,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -354,7 +354,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -365,7 +365,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -376,7 +376,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -387,7 +387,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -398,7 +398,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -409,7 +409,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -420,7 +420,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -431,7 +431,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -442,7 +442,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -453,7 +453,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -464,7 +464,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -475,7 +475,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -486,7 +486,7 @@ impl Vecs {
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
price,
states_path,
true,
)?,
@@ -546,12 +546,12 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.as_mut_vecs().into_iter().try_for_each(|(_, v)| {
v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
v.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)
})
}
@@ -560,7 +560,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -568,13 +568,13 @@ impl Vecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.0.as_boxed_mut_vecs().into_iter().try_for_each(|v| {
v.into_iter().try_for_each(|(_, v)| {
v.compute_rest_part2(
indexer,
indexes,
fetched,
price,
starting_indexes,
market,
height_to_supply,

4
crates/brk_computer/src/stateful/addresstype_to_addresscount.rs
View File

@@ -1,11 +1,11 @@
use brk_core::{ByAddressType, Height};
use brk_structs::{ByAddressType, Height};
use brk_vecs::VecIterator;
use derive_deref::{Deref, DerefMut};
use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(ByAddressType<usize>);
pub struct AddressTypeToAddressCount(ByAddressType<u64>);
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {

12
crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs
View File

@@ -1,15 +1,15 @@
use brk_core::{ByAddressType, Height, Result, StoredUsize};
use brk_exit::Exit;
use brk_vecs::EagerVec;
use brk_error::Result;
use brk_structs::{ByAddressType, Height, StoredU64};
use brk_vecs::{EagerVec, Exit, GenericStoredVec};
use derive_deref::{Deref, DerefMut};
use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredUsize>>);
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredU64>>);
impl From<ByAddressType<EagerVec<Height, StoredUsize>>> for AddressTypeToHeightToAddressCount {
fn from(value: ByAddressType<EagerVec<Height, StoredUsize>>) -> Self {
impl From<ByAddressType<EagerVec<Height, StoredU64>>> for AddressTypeToHeightToAddressCount {
fn from(value: ByAddressType<EagerVec<Height, StoredU64>>) -> Self {
Self(value)
}
}

16
crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs
View File

@@ -1,6 +1,6 @@
use brk_core::{ByAddressType, StoredUsize};
use brk_exit::Exit;
use brk_vecs::AnyCollectableVec;
use brk_error::Result;
use brk_structs::{ByAddressType, StoredU64};
use brk_vecs::{AnyCollectableVec, Exit};
use derive_deref::{Deref, DerefMut};
use crate::{
@@ -9,12 +9,10 @@ use crate::{
};
#[derive(Clone, Deref, DerefMut)]
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredUsize>>);
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredU64>>);
impl From<ByAddressType<ComputedVecsFromHeight<StoredUsize>>>
for AddressTypeToIndexesToAddressCount
{
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredUsize>>) -> Self {
impl From<ByAddressType<ComputedVecsFromHeight<StoredU64>>> for AddressTypeToIndexesToAddressCount {
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredU64>>) -> Self {
Self(value)
}
}
@@ -27,7 +25,7 @@ impl AddressTypeToIndexesToAddressCount {
starting_indexes: &Indexes,
exit: &Exit,
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.p2pk65.compute_rest(
indexes,
starting_indexes,

2
crates/brk_computer/src/stateful/addresstype_to_typeindex_set.rs
View File

@@ -1,6 +1,6 @@
use std::collections::BTreeSet;
use brk_core::TypeIndex;
use brk_structs::TypeIndex;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;

2
crates/brk_computer/src/stateful/addresstype_to_typeindex_tree.rs
View File

@@ -1,6 +1,6 @@
use std::{collections::BTreeMap, mem};
use brk_core::TypeIndex;
use brk_structs::TypeIndex;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;

138
crates/brk_computer/src/stateful/common.rs
View File

@@ -1,22 +1,22 @@
use std::sync::Arc;
use brk_core::{
Bitcoin, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredF64, StoredUsize, Version,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Bitcoin, DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64, Version,
};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec,
File, Format, VecIterator,
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation,
EagerVec, Exit, File, Format, GenericStoredVec, VecIterator,
};
use crate::{
Indexes, fetched,
Indexes,
grouped::{
ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions,
},
indexes, market,
indexes, market, price,
states::CohortState,
};
@@ -27,7 +27,7 @@ pub struct Vecs {
// Cumulative
pub height_to_realized_cap: Option<EagerVec<Height, Dollars>>,
pub height_to_supply: EagerVec<Height, Sats>,
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub height_to_utxo_count: EagerVec<Height, StoredU64>,
// Single
pub dateindex_to_supply_even: Option<EagerVec<DateIndex, Sats>>,
pub dateindex_to_supply_in_loss: Option<EagerVec<DateIndex, Sats>>,
@@ -68,7 +68,7 @@ pub struct Vecs {
pub indexes_to_realized_value: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_supply_value: ComputedHeightValueVecs,
pub indexes_to_supply: ComputedValueVecsFromDateIndex,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_unrealized_profit: Option<ComputedVecsFromDateIndex<Dollars>>,
@@ -134,11 +134,11 @@ impl Vecs {
format: Format,
version: Version,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
compute_relative_to_all: bool,
ratio_extended: bool,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
) -> Result<Self> {
let compute_dollars = price.is_some();
// let prefix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{s}_{name}"));
@@ -1368,7 +1368,7 @@ impl Vecs {
self.height_to_utxo_count.forced_push_at(
height,
StoredUsize::from(state.supply.utxos),
StoredU64::from(state.supply.utxos),
exit,
)?;
@@ -1899,12 +1899,12 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.height_to_supply_value.compute_rest(
fetched,
price,
starting_indexes,
exit,
Some(&self.height_to_supply),
@@ -1913,7 +1913,7 @@ impl Vecs {
self.indexes_to_supply.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|v, _, indexes, starting_indexes, exit| {
@@ -1925,14 +1925,15 @@ impl Vecs {
&indexes.dateindex_to_first_height,
|(i, height, ..)| {
let count = dateindex_to_height_count_iter.unwrap_get_inner(i);
if count == StoredUsize::default() {
if count == StoredU64::default() {
unreachable!()
}
let supply = height_to_supply_iter.unwrap_get_inner(height + (*count - 1));
(i, supply)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1946,7 +1947,7 @@ impl Vecs {
self.height_to_halved_supply_value.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
@@ -1955,14 +1956,15 @@ impl Vecs {
&self.height_to_supply,
|(h, v, ..)| (h, v / 2),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_halved_supply.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
@@ -1971,7 +1973,8 @@ impl Vecs {
self.indexes_to_supply.sats.dateindex.as_ref().unwrap(),
|(i, sats, ..)| (i, sats / 2),
exit,
)
)?;
Ok(())
},
)?;
@@ -1986,7 +1989,8 @@ impl Vecs {
&self.height_to_satblocks_destroyed,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)
)?;
Ok(())
},
)?;
@@ -2001,7 +2005,8 @@ impl Vecs {
&self.height_to_satdays_destroyed,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)
)?;
Ok(())
},
)?;
@@ -2013,7 +2018,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -2021,7 +2026,7 @@ impl Vecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(v) = self
.indexes_to_supply_relative_to_circulating_supply
.as_mut()
@@ -2037,7 +2042,8 @@ impl Vecs {
&self.height_to_supply_value.bitcoin,
height_to_supply,
exit,
)
)?;
Ok(())
},
)?;
}
@@ -2064,7 +2070,8 @@ impl Vecs {
self.height_to_realized_cap.as_ref().unwrap(),
&self.height_to_supply_value.bitcoin,
exit,
)
)?;
Ok(())
},
)?;
@@ -2074,7 +2081,7 @@ impl Vecs {
.compute_rest(
indexer,
indexes,
fetched.as_ref().unwrap(),
price.as_ref().unwrap(),
starting_indexes,
exit,
Some(
@@ -2120,7 +2127,8 @@ impl Vecs {
self.height_to_realized_loss.as_ref().unwrap(),
|(i, v, ..)| (i, v * -1_i64),
exit,
)
)?;
Ok(())
},
)?;
@@ -2182,7 +2190,8 @@ impl Vecs {
.unwrap_last(),
30,
exit,
)
)?;
Ok(())
},
)?;
@@ -2200,7 +2209,8 @@ impl Vecs {
self.height_to_realized_profit.as_ref().unwrap(),
self.height_to_realized_loss.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2218,7 +2228,8 @@ impl Vecs {
self.height_to_realized_profit.as_ref().unwrap(),
self.height_to_realized_loss.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2282,7 +2293,7 @@ impl Vecs {
.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.dateindex_to_supply_in_profit.as_ref().unwrap()),
@@ -2293,7 +2304,7 @@ impl Vecs {
.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.dateindex_to_supply_in_loss.as_ref().unwrap()),
@@ -2301,7 +2312,7 @@ impl Vecs {
self.indexes_to_supply_even.as_mut().unwrap().compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.dateindex_to_supply_even.as_ref().unwrap()),
@@ -2367,7 +2378,8 @@ impl Vecs {
self.dateindex_to_unrealized_loss.as_ref().unwrap(),
|(h, v, ..)| (h, v * -1_i64),
exit,
)
)?;
Ok(())
},
)?;
self.height_to_net_unrealized_profit_and_loss
@@ -2394,7 +2406,8 @@ impl Vecs {
self.dateindex_to_unrealized_profit.as_ref().unwrap(),
self.dateindex_to_unrealized_loss.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.height_to_net_unrealized_profit_and_loss_relative_to_market_cap
@@ -2427,7 +2440,8 @@ impl Vecs {
.unwrap(),
market.indexes_to_marketcap.dateindex.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2445,7 +2459,8 @@ impl Vecs {
self.height_to_realized_profit.as_ref().unwrap(),
*height_to_realized_cap.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2463,7 +2478,8 @@ impl Vecs {
self.height_to_realized_loss.as_ref().unwrap(),
*height_to_realized_cap.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2486,7 +2502,8 @@ impl Vecs {
.unwrap(),
*height_to_realized_cap.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2494,7 +2511,7 @@ impl Vecs {
.as_mut()
.unwrap()
.compute_rest(
fetched,
price,
starting_indexes,
exit,
Some(self.height_to_supply_even.as_ref().unwrap()),
@@ -2503,7 +2520,7 @@ impl Vecs {
.as_mut()
.unwrap()
.compute_rest(
fetched,
price,
starting_indexes,
exit,
Some(self.height_to_supply_in_loss.as_ref().unwrap()),
@@ -2512,7 +2529,7 @@ impl Vecs {
.as_mut()
.unwrap()
.compute_rest(
fetched,
price,
starting_indexes,
exit,
Some(self.height_to_supply_in_profit.as_ref().unwrap()),
@@ -2572,7 +2589,8 @@ impl Vecs {
.unwrap(),
self.indexes_to_supply.bitcoin.dateindex.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_supply_in_loss_relative_to_own_supply
@@ -2595,7 +2613,8 @@ impl Vecs {
.unwrap(),
self.indexes_to_supply.bitcoin.dateindex.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_supply_in_profit_relative_to_own_supply
@@ -2618,7 +2637,8 @@ impl Vecs {
.unwrap(),
self.indexes_to_supply.bitcoin.dateindex.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2640,7 +2660,8 @@ impl Vecs {
.unwrap_cumulative(),
30,
exit,
)
)?;
Ok(())
},
)?;
@@ -2658,7 +2679,8 @@ impl Vecs {
self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(),
*dateindex_to_realized_cap.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2676,7 +2698,8 @@ impl Vecs {
self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(),
market.indexes_to_marketcap.dateindex.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -2736,7 +2759,8 @@ impl Vecs {
.unwrap(),
dateindex_to_supply,
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_supply_in_loss_relative_to_circulating_supply
@@ -2759,7 +2783,8 @@ impl Vecs {
.unwrap(),
dateindex_to_supply,
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_supply_in_profit_relative_to_circulating_supply
@@ -2782,7 +2807,8 @@ impl Vecs {
.unwrap(),
dateindex_to_supply,
exit,
)
)?;
Ok(())
},
)?;
}

2
crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs
View File

@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_core::Height;
use brk_structs::Height;
use derive_deref::{Deref, DerefMut};
use crate::stateful::AddressTypeToVec;

368
crates/brk_computer/src/stateful/mod.rs
View File

@@ -1,31 +1,26 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, sync::Arc, thread};
use brk_core::{
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
AnyAddressDataIndexEnum, AnyAddressIndex, ByAddressType, ByAnyAddress, CheckedSub, DateIndex,
Dollars, EmptyAddressData, EmptyAddressIndex, Height, InputIndex, LoadedAddressData,
LoadedAddressIndex, OutputIndex, OutputType, P2AAddressIndex, P2PK33AddressIndex,
P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex,
P2WSHAddressIndex, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version,
P2WSHAddressIndex, Sats, StoredU64, Timestamp, TypeIndex, Version,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vecs::{
AnyCollectableVec, AnyStampedVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format,
GenericStoredVec, Reader, Stamp, StampedVec, StoredIndex, StoredVec, VecIterator,
AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, Exit, File,
Format, GenericStoredVec, RawVec, Reader, Stamp, StoredIndex, VecIterator,
};
use log::info;
use rayon::prelude::*;
use crate::{
BlockState, SupplyState, Transacted,
grouped::{ComputedVecsFromHeight, Source},
market,
};
use super::{
Indexes, fetched,
BlockState, Indexes, SupplyState, Transacted,
grouped::{ComputedValueVecsFromHeight, VecBuilderOptions},
indexes, transactions,
grouped::{ComputedVecsFromHeight, Source},
indexes, market, price, transactions,
};
mod address_cohort;
@@ -58,7 +53,9 @@ const VERSION: Version = Version::new(21);
#[derive(Clone)]
pub struct Vecs {
pub chain_state: StoredVec<Height, SupplyState>,
file: Arc<File>,
pub chain_state: RawVec<Height, SupplyState>,
pub height_to_unspendable_supply: EagerVec<Height, Sats>,
pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight,
@@ -71,51 +68,52 @@ pub struct Vecs {
pub utxo_cohorts: utxo_cohorts::Vecs,
pub address_cohorts: address_cohorts::Vecs,
pub p2pk33addressindex_to_anyaddressindex: StampedVec<P2PK33AddressIndex, AnyAddressIndex>,
pub p2pk65addressindex_to_anyaddressindex: StampedVec<P2PK65AddressIndex, AnyAddressIndex>,
pub p2pkhaddressindex_to_anyaddressindex: StampedVec<P2PKHAddressIndex, AnyAddressIndex>,
pub p2shaddressindex_to_anyaddressindex: StampedVec<P2SHAddressIndex, AnyAddressIndex>,
pub p2traddressindex_to_anyaddressindex: StampedVec<P2TRAddressIndex, AnyAddressIndex>,
pub p2wpkhaddressindex_to_anyaddressindex: StampedVec<P2WPKHAddressIndex, AnyAddressIndex>,
pub p2wshaddressindex_to_anyaddressindex: StampedVec<P2WSHAddressIndex, AnyAddressIndex>,
pub p2aaddressindex_to_anyaddressindex: StampedVec<P2AAddressIndex, AnyAddressIndex>,
pub loadedaddressindex_to_loadedaddressdata: StampedVec<LoadedAddressIndex, LoadedAddressData>,
pub emptyaddressindex_to_emptyaddressdata: StampedVec<EmptyAddressIndex, EmptyAddressData>,
pub p2pk33addressindex_to_anyaddressindex: RawVec<P2PK33AddressIndex, AnyAddressIndex>,
pub p2pk65addressindex_to_anyaddressindex: RawVec<P2PK65AddressIndex, AnyAddressIndex>,
pub p2pkhaddressindex_to_anyaddressindex: RawVec<P2PKHAddressIndex, AnyAddressIndex>,
pub p2shaddressindex_to_anyaddressindex: RawVec<P2SHAddressIndex, AnyAddressIndex>,
pub p2traddressindex_to_anyaddressindex: RawVec<P2TRAddressIndex, AnyAddressIndex>,
pub p2wpkhaddressindex_to_anyaddressindex: RawVec<P2WPKHAddressIndex, AnyAddressIndex>,
pub p2wshaddressindex_to_anyaddressindex: RawVec<P2WSHAddressIndex, AnyAddressIndex>,
pub p2aaddressindex_to_anyaddressindex: RawVec<P2AAddressIndex, AnyAddressIndex>,
pub loadedaddressindex_to_loadedaddressdata: RawVec<LoadedAddressIndex, LoadedAddressData>,
pub emptyaddressindex_to_emptyaddressdata: RawVec<EmptyAddressIndex, EmptyAddressData>,
pub indexes_to_address_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_empty_address_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_address_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_empty_address_count: ComputedVecsFromHeight<StoredU64>,
}
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
states_path: &Path,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("stateful"))?);
let chain_file = Arc::new(File::open(&states_path.join("chain"))?);
let compute_dollars = price.is_some();
let chain_file = Arc::new(File::open(&parent.join("chain"))?);
Ok(Self {
chain_state: StoredVec::forced_import(
chain_state: RawVec::forced_import(
&chain_file,
"chain",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
file,
&file,
"unspendable_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
file,
&file,
"unspendable_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -126,13 +124,13 @@ impl Vecs {
indexes,
)?,
height_to_opreturn_supply: EagerVec::forced_import(
file,
&file,
"opreturn_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
file,
&file,
"opreturn_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -143,7 +141,7 @@ impl Vecs {
indexes,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -153,7 +151,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"empty_address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -165,49 +163,49 @@ impl Vecs {
addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
file,
&file,
"p2pk65_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
file,
&file,
"p2pk33_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
file,
&file,
"p2pkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
file,
&file,
"p2sh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
file,
&file,
"p2wpkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
file,
&file,
"p2wsh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
file,
&file,
"p2tr_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
file,
&file,
"p2a_address_count",
version + VERSION + Version::ZERO,
format,
@@ -217,49 +215,49 @@ impl Vecs {
addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
file,
&file,
"p2pk65_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
file,
&file,
"p2pk33_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
file,
&file,
"p2pkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
file,
&file,
"p2sh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
file,
&file,
"p2wpkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
file,
&file,
"p2wsh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
file,
&file,
"p2tr_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
file,
&file,
"p2a_empty_address_count",
version + VERSION + Version::ZERO,
format,
@@ -269,7 +267,7 @@ impl Vecs {
addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pk65_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -279,7 +277,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pk33_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -289,7 +287,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -299,7 +297,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2sh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -309,7 +307,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2wpkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -319,7 +317,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2wsh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -329,7 +327,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2tr_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -339,7 +337,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2a_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -353,7 +351,7 @@ impl Vecs {
addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pk65_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -363,7 +361,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pk33_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -373,7 +371,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -383,7 +381,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2sh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -393,7 +391,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2wpkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -403,7 +401,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2wsh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -413,7 +411,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2tr_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -423,7 +421,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2a_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -435,85 +433,77 @@ impl Vecs {
},
),
utxo_cohorts: utxo_cohorts::Vecs::forced_import(
file,
&file,
version,
computation,
format,
indexes,
fetched,
price,
states_path,
)?,
address_cohorts: address_cohorts::Vecs::forced_import(
file,
&file,
version,
computation,
format,
indexes,
fetched,
price,
states_path,
)?,
p2aaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2aaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk33addressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2pk33addressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk65addressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2pk65addressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pkhaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2pkhaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2shaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2shaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2traddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2traddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wpkhaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2wpkhaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wshaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
p2wshaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
loadedaddressindex_to_loadedaddressdata: StampedVec::forced_import(
file,
loadedaddressindex_to_loadedaddressdata: RawVec::forced_import(
&file,
"loadedaddressdata",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
emptyaddressindex_to_emptyaddressdata: StampedVec::forced_import(
file,
emptyaddressindex_to_emptyaddressdata: RawVec::forced_import(
&file,
"emptyaddressdata",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
file,
})
}
@@ -523,12 +513,12 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
transactions: &transactions::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
market: &market::Vecs,
// Must take ownership as its indexes will be updated for this specific function
starting_indexes: &mut Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
let height_to_first_outputindex = &indexer.vecs.height_to_first_outputindex;
let height_to_first_inputindex = &indexer.vecs.height_to_first_inputindex;
let height_to_first_p2aaddressindex = &indexer.vecs.height_to_first_p2aaddressindex;
@@ -554,12 +544,12 @@ impl Vecs {
.height
.as_ref()
.unwrap();
let height_to_close = fetched
let height_to_close = price
.as_ref()
.map(|fetched| &fetched.chainindexes_to_close.height);
let dateindex_to_close = fetched
.map(|price| &price.chainindexes_to_close.height);
let dateindex_to_close = price
.as_ref()
.map(|fetched| fetched.timeindexes_to_close.dateindex.as_ref().unwrap());
.map(|price| price.timeindexes_to_close.dateindex.as_ref().unwrap());
let height_to_date_fixed = &indexes.height_to_date_fixed;
let dateindex_to_first_height = &indexes.dateindex_to_first_height;
let dateindex_to_height_count = &indexes.dateindex_to_height_count;
@@ -632,56 +622,16 @@ impl Vecs {
.unwrap_or_default(),
)
.min(chain_state_starting_height)
.min(
Height::from(u64::from(
self.p2pk33addressindex_to_anyaddressindex.stamp(),
))
.incremented(),
)
.min(
Height::from(u64::from(
self.p2pk65addressindex_to_anyaddressindex.stamp(),
))
.incremented(),
)
.min(
Height::from(u64::from(self.p2pkhaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
Height::from(u64::from(self.p2shaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
Height::from(u64::from(self.p2traddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
Height::from(u64::from(
self.p2wpkhaddressindex_to_anyaddressindex.stamp(),
))
.incremented(),
)
.min(
Height::from(u64::from(self.p2wshaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
Height::from(u64::from(self.p2aaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
Height::from(u64::from(
self.loadedaddressindex_to_loadedaddressdata.stamp(),
))
.incremented(),
)
.min(
Height::from(u64::from(
self.emptyaddressindex_to_emptyaddressdata.stamp(),
))
.incremented(),
)
.min(Height::from(self.p2pk33addressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.p2pk65addressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.p2pkhaddressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.p2shaddressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.p2traddressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.p2wpkhaddressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.p2wshaddressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.p2aaddressindex_to_anyaddressindex.stamp()).incremented())
.min(Height::from(self.loadedaddressindex_to_loadedaddressdata.stamp()).incremented())
.min(Height::from(self.emptyaddressindex_to_emptyaddressdata.stamp()).incremented())
.min(Height::from(self.height_to_unspendable_supply.len()))
.min(Height::from(self.height_to_opreturn_supply.len()))
.cmp(&chain_state_starting_height)
@@ -745,7 +695,7 @@ impl Vecs {
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
};
let last_height = Height::from(u64::from(indexer.vecs.height_to_blockhash.stamp()));
let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp());
if starting_height <= last_height {
let inputindex_to_outputindex_reader = inputindex_to_outputindex.create_reader();
@@ -836,7 +786,7 @@ impl Vecs {
(height.unwrap_to_usize()..height_to_date_fixed.len())
.map(Height::from)
.try_for_each(|_height| -> color_eyre::Result<()> {
.try_for_each(|_height| -> Result<()> {
height = _height;
info!("Processing chain at {height}...");
@@ -903,7 +853,7 @@ impl Vecs {
.tick_tock_next_block(&chain_state, timestamp);
});
let (transacted, addresstype_to_typedindex_to_received_data, receiving_addresstype_to_typeindex_to_addressdatawithsource) = (first_outputindex..first_outputindex + *output_count)
let (transacted, addresstype_to_typedindex_to_received_data, receiving_addresstype_to_typeindex_to_addressdatawithsource) = (first_outputindex..first_outputindex + usize::from(output_count))
.into_par_iter()
.map(OutputIndex::from)
.map(|outputindex| {
@@ -987,7 +937,7 @@ impl Vecs {
);
// Skip coinbase
let (height_to_sent, addresstype_to_typedindex_to_sent_data, sending_addresstype_to_typeindex_to_addressdatawithsource) = (first_inputindex + 1..first_inputindex + *input_count)
let (height_to_sent, addresstype_to_typedindex_to_sent_data, sending_addresstype_to_typeindex_to_addressdatawithsource) = (first_inputindex + 1..first_inputindex + usize::from(input_count))
.into_par_iter()
.map(InputIndex::from)
.map(|inputindex| {
@@ -1287,7 +1237,8 @@ impl Vecs {
.map(|(_, v)| v)
.collect::<Vec<_>>(),
exit,
)
)?;
Ok(())
},
)?;
@@ -1306,14 +1257,15 @@ impl Vecs {
.map(|(_, v)| v)
.collect::<Vec<_>>(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_unspendable_supply.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(&self.height_to_unspendable_supply),
@@ -1321,7 +1273,7 @@ impl Vecs {
self.indexes_to_opreturn_supply.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(&self.height_to_opreturn_supply),
@@ -1342,15 +1294,10 @@ impl Vecs {
)?;
self.utxo_cohorts
.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)?;
.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)?;
self.address_cohorts.compute_rest_part1(
indexer,
indexes,
fetched,
starting_indexes,
exit,
)?;
self.address_cohorts
.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)?;
info!("Computing rest part 2...");
@@ -1384,7 +1331,7 @@ impl Vecs {
self.utxo_cohorts.compute_rest_part2(
indexer,
indexes,
fetched,
price,
starting_indexes,
market,
height_to_supply,
@@ -1397,7 +1344,7 @@ impl Vecs {
self.address_cohorts.compute_rest_part2(
indexer,
indexes,
fetched,
price,
starting_indexes,
market,
height_to_supply,
@@ -1407,6 +1354,8 @@ impl Vecs {
exit,
)?;
self.file.flush()?;
self.file.punch_holes()?;
Ok(())
}
@@ -1423,16 +1372,16 @@ impl Vecs {
>,
addresstypeindex_to_anyaddressindex_reader_opt: &ByAddressType<Option<Reader>>,
anyaddressindex_to_anyaddressdata_reader_opt: &ByAnyAddress<Option<Reader>>,
p2pk33addressindex_to_anyaddressindex: &StampedVec<P2PK33AddressIndex, AnyAddressIndex>,
p2pk65addressindex_to_anyaddressindex: &StampedVec<P2PK65AddressIndex, AnyAddressIndex>,
p2pkhaddressindex_to_anyaddressindex: &StampedVec<P2PKHAddressIndex, AnyAddressIndex>,
p2shaddressindex_to_anyaddressindex: &StampedVec<P2SHAddressIndex, AnyAddressIndex>,
p2traddressindex_to_anyaddressindex: &StampedVec<P2TRAddressIndex, AnyAddressIndex>,
p2wpkhaddressindex_to_anyaddressindex: &StampedVec<P2WPKHAddressIndex, AnyAddressIndex>,
p2wshaddressindex_to_anyaddressindex: &StampedVec<P2WSHAddressIndex, AnyAddressIndex>,
p2aaddressindex_to_anyaddressindex: &StampedVec<P2AAddressIndex, AnyAddressIndex>,
loadedaddressindex_to_loadedaddressdata: &StampedVec<LoadedAddressIndex, LoadedAddressData>,
emptyaddressindex_to_emptyaddressdata: &StampedVec<EmptyAddressIndex, EmptyAddressData>,
p2pk33addressindex_to_anyaddressindex: &RawVec<P2PK33AddressIndex, AnyAddressIndex>,
p2pk65addressindex_to_anyaddressindex: &RawVec<P2PK65AddressIndex, AnyAddressIndex>,
p2pkhaddressindex_to_anyaddressindex: &RawVec<P2PKHAddressIndex, AnyAddressIndex>,
p2shaddressindex_to_anyaddressindex: &RawVec<P2SHAddressIndex, AnyAddressIndex>,
p2traddressindex_to_anyaddressindex: &RawVec<P2TRAddressIndex, AnyAddressIndex>,
p2wpkhaddressindex_to_anyaddressindex: &RawVec<P2WPKHAddressIndex, AnyAddressIndex>,
p2wshaddressindex_to_anyaddressindex: &RawVec<P2WSHAddressIndex, AnyAddressIndex>,
p2aaddressindex_to_anyaddressindex: &RawVec<P2AAddressIndex, AnyAddressIndex>,
loadedaddressindex_to_loadedaddressdata: &RawVec<LoadedAddressIndex, LoadedAddressData>,
emptyaddressindex_to_emptyaddressdata: &RawVec<EmptyAddressIndex, EmptyAddressData>,
) -> Option<WithAddressDataSource<LoadedAddressData>> {
if *first_addressindexes.get(address_type).unwrap() <= typeindex {
return Some(WithAddressDataSource::New(LoadedAddressData::default()));
@@ -1643,7 +1592,8 @@ impl Vecs {
emptyaddressdata,
)) => self
.emptyaddressindex_to_emptyaddressdata
.update(emptyaddressindex, emptyaddressdata),
.update(emptyaddressindex, emptyaddressdata)
.map_err(|e| e.into()),
WithAddressDataSource::FromLoadedAddressDataVec((
loadedaddressindex,
emptyaddressdata,
@@ -1695,7 +1645,8 @@ impl Vecs {
loadedaddressdata,
)) => self
.loadedaddressindex_to_loadedaddressdata
.update(loadedaddressindex, loadedaddressdata),
.update(loadedaddressindex, loadedaddressdata)
.map_err(|e| e.into()),
WithAddressDataSource::FromEmptyAddressDataVec((
emptyaddressindex,
loadedaddressdata,
@@ -1753,30 +1704,31 @@ impl Vecs {
.p2aaddressindex_to_anyaddressindex
.update_or_push(typeindex.into(), anyaddressindex),
_ => unreachable!(),
}
}?;
Ok(())
})
})?;
self.p2pk33addressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.p2pk65addressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.p2pkhaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.p2shaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.p2traddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.p2wpkhaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.p2wshaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.p2aaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.loadedaddressindex_to_loadedaddressdata
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.emptyaddressindex_to_emptyaddressdata
.flush(Stamp::from(u64::from(height)))?;
.stamped_flush(Stamp::from(height))?;
self.chain_state.truncate_if_needed(Height::ZERO)?;
chain_state.iter().for_each(|block_state| {
@@ -1838,8 +1790,8 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
WithAddressDataSource<EmptyAddressData>,
>,
price: Option<Dollars>,
addresstype_to_address_count: &mut ByAddressType<usize>,
addresstype_to_empty_address_count: &mut ByAddressType<usize>,
addresstype_to_address_count: &mut ByAddressType<u64>,
addresstype_to_empty_address_count: &mut ByAddressType<u64>,
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexTree<
WithAddressDataSource<LoadedAddressData>,
>,
@@ -1928,9 +1880,9 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
WithAddressDataSource<EmptyAddressData>,
>,
price: Option<Dollars>,
addresstype_to_address_count: &mut ByAddressType<usize>,
addresstype_to_empty_address_count: &mut ByAddressType<usize>,
height_to_close_vec: Option<&Vec<brk_core::Close<Dollars>>>,
addresstype_to_address_count: &mut ByAddressType<u64>,
addresstype_to_empty_address_count: &mut ByAddressType<u64>,
height_to_close_vec: Option<&Vec<brk_structs::Close<Dollars>>>,
height_to_timestamp_fixed_vec: &[Timestamp],
height: Height,
timestamp: Timestamp,

22
crates/brk_computer/src/stateful/range_map.rs
View File

@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_vecs::{StampedVec, StoredIndex, StoredType};
use brk_vecs::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw};
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
@@ -20,12 +20,26 @@ where
}
}
impl<I, T> From<&StampedVec<I, T>> for RangeMap<T, I>
impl<I, T> From<&RawVec<I, T>> for RangeMap<T, I>
where
I: StoredIndex,
T: StoredIndex + StoredType,
T: StoredIndex + StoredRaw,
{
fn from(vec: &StampedVec<I, T>) -> Self {
fn from(vec: &RawVec<I, T>) -> Self {
Self(
vec.into_iter()
.map(|(i, v)| (v.into_owned(), i))
.collect::<BTreeMap<_, _>>(),
)
}
}
impl<I, T> From<&CompressedVec<I, T>> for RangeMap<T, I>
where
I: StoredIndex,
T: StoredIndex + StoredCompressed,
{
fn from(vec: &CompressedVec<I, T>) -> Self {
Self(
vec.into_iter()
.map(|(i, v)| (v.into_owned(), i))

16
crates/brk_computer/src/stateful/trait.rs
View File

@@ -1,9 +1,9 @@
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, AnyIterableVec};
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Exit};
use crate::{Indexes, fetched, indexes, market};
use crate::{Indexes, indexes, market, price};
pub trait DynCohortVecs: Send + Sync {
fn starting_height(&self) -> Height;
@@ -30,10 +30,10 @@ pub trait DynCohortVecs: Send + Sync {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()>;
) -> Result<()>;
fn vecs(&self) -> Vec<&dyn AnyCollectableVec>;
}
@@ -51,7 +51,7 @@ pub trait CohortVecs: DynCohortVecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -59,5 +59,5 @@ pub trait CohortVecs: DynCohortVecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()>;
) -> Result<()>;
}

28
crates/brk_computer/src/stateful/utxo_cohort.rs
View File

@@ -1,12 +1,12 @@
use std::{ops::Deref, path::Path, sync::Arc};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, File, Format};
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, Exit, File, Format};
use crate::{
Indexes, UTXOCohortState, fetched, indexes, market,
Indexes, UTXOCohortState, indexes, market, price,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
@@ -31,12 +31,12 @@ impl Vecs {
format: Format,
version: Version,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
states_path: &Path,
compute_relative_to_all: bool,
ratio_extended: bool,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
) -> Result<Self> {
let compute_dollars = price.is_some();
Ok(Self {
starting_height: Height::ZERO,
@@ -54,7 +54,7 @@ impl Vecs {
format,
version,
indexes,
fetched,
price,
compute_relative_to_all,
ratio_extended,
)?,
@@ -123,12 +123,12 @@ impl DynCohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.inner
.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)
}
fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -155,7 +155,7 @@ impl CohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -163,11 +163,11 @@ impl CohortVecs for Vecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.inner.compute_rest_part2(
indexer,
indexes,
fetched,
price,
starting_indexes,
market,
height_to_supply,

260
crates/brk_computer/src/stateful/utxo_cohorts.rs
View File

File diff suppressed because it is too large Load Diff

2
crates/brk_computer/src/stateful/withaddressdatasource.rs
View File

@@ -1,4 +1,4 @@
use brk_core::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
use brk_structs::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
#[derive(Debug)]
pub enum WithAddressDataSource<T> {

2
crates/brk_computer/src/states/block.rs
View File

@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_core::{Dollars, Timestamp};
use brk_structs::{Dollars, Timestamp};
use super::SupplyState;

13
crates/brk_computer/src/states/cohorts/address.rs
View File

@@ -1,6 +1,7 @@
use std::path::Path;
use brk_core::{Dollars, Height, LoadedAddressData, Result, Sats};
use brk_error::Result;
use brk_structs::{Dollars, Height, LoadedAddressData, Sats};
use crate::SupplyState;
@@ -8,7 +9,7 @@ use super::CohortState;
#[derive(Clone)]
pub struct AddressCohortState {
pub address_count: usize,
pub address_count: u64,
pub inner: CohortState,
}
@@ -47,14 +48,14 @@ impl AddressCohortState {
let prev_realized_price = compute_price.then(|| addressdata.realized_price());
let prev_supply_state = SupplyState {
utxos: addressdata.outputs_len as usize,
utxos: addressdata.outputs_len as u64,
value: addressdata.amount(),
};
addressdata.send(value, prev_price)?;
let supply_state = SupplyState {
utxos: addressdata.outputs_len as usize,
utxos: addressdata.outputs_len as u64,
value: addressdata.amount(),
};
@@ -82,14 +83,14 @@ impl AddressCohortState {
let prev_realized_price = compute_price.then(|| address_data.realized_price());
let prev_supply_state = SupplyState {
utxos: address_data.outputs_len as usize,
utxos: address_data.outputs_len as u64,
value: address_data.amount(),
};
address_data.receive(value, price);
let supply_state = SupplyState {
utxos: address_data.outputs_len as usize,
utxos: address_data.outputs_len as u64,
value: address_data.amount(),
};

3
crates/brk_computer/src/states/cohorts/common.rs
View File

@@ -1,6 +1,7 @@
use std::{cmp::Ordering, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Result, Sats};
use brk_error::Result;
use brk_structs::{CheckedSub, Dollars, Height, Sats};
use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState};

3
crates/brk_computer/src/states/cohorts/utxo.rs
View File

@@ -1,6 +1,7 @@
use std::path::Path;
use brk_core::{Height, Result};
use brk_error::Result;
use brk_structs::Height;
use derive_deref::{Deref, DerefMut};
use super::CohortState;

3
crates/brk_computer/src/states/price_to_amount.rs
View File

@@ -5,7 +5,8 @@ use std::{
path::{Path, PathBuf},
};
use brk_core::{Dollars, Error, Height, Result, Sats};
use brk_error::{Error, Result};
use brk_structs::{Dollars, Height, Sats};
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, IntoBytes};

2
crates/brk_computer/src/states/realized.rs
View File

@@ -1,6 +1,6 @@
use std::cmp::Ordering;
use brk_core::{CheckedSub, Dollars};
use brk_structs::{CheckedSub, Dollars};
use super::SupplyState;

6
crates/brk_computer/src/states/supply.rs
View File

@@ -1,12 +1,12 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_core::{CheckedSub, LoadedAddressData, Sats};
use brk_structs::{CheckedSub, LoadedAddressData, Sats};
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
pub struct SupplyState {
pub utxos: usize,
pub utxos: u64,
pub value: Sats,
}
@@ -43,7 +43,7 @@ impl SubAssign<&SupplyState> for SupplyState {
impl From<&LoadedAddressData> for SupplyState {
fn from(value: &LoadedAddressData) -> Self {
Self {
utxos: value.outputs_len as usize,
utxos: value.outputs_len as u64,
value: value.amount(),
}
}

2
crates/brk_computer/src/states/transacted.rs
View File

@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign};
use brk_core::{ByAmountRange, GroupedByType, OutputType, Sats};
use brk_structs::{ByAmountRange, GroupedByType, OutputType, Sats};
use super::SupplyState;

2
crates/brk_computer/src/states/unrealized.rs
View File

@@ -1,4 +1,4 @@
use brk_core::{Dollars, Sats};
use brk_structs::{Dollars, Sats};
#[derive(Debug, Default, Clone)]
pub struct UnrealizedState {

316
crates/brk_computer/src/traits.rs Normal file
View File

@@ -0,0 +1,316 @@
use brk_error::Result;
use brk_structs::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32};
use brk_vecs::{
AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, StoredIndex,
VecIterator, Version,
};
const DCA_AMOUNT: Dollars = Dollars::mint(100.0);
pub trait ComputeDCAStackViaLen {
fn compute_dca_stack_via_len(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
len: usize,
exit: &Exit,
) -> Result<()>;
fn compute_dca_stack_via_from(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
from: DateIndex,
exit: &Exit,
) -> Result<()>;
}
impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
fn compute_dca_stack_via_len(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner_version() + closes.version(),
)?;
let mut other_iter = closes.iter();
let mut prev = None;
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
let price = *closes.into_owned();
let i_usize = i.unwrap_to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1));
}
}
let mut stack = Sats::ZERO;
if price != Dollars::ZERO {
stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
if i_usize >= len {
let prev_price = *other_iter.unwrap_get_inner_(i_usize - len);
if prev_price != Dollars::ZERO {
stack = stack
.checked_sub(Sats::from(Bitcoin::from(DCA_AMOUNT / prev_price)))
.unwrap();
}
}
}
prev.replace(stack);
self.forced_push_at(i, stack, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
fn compute_dca_stack_via_from(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner_version() + closes.version(),
)?;
let mut prev = None;
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
let price = *closes.into_owned();
let i_usize = i.unwrap_to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1));
}
}
let mut stack = Sats::ZERO;
if price != Dollars::ZERO && i >= from {
stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
}
prev.replace(stack);
self.forced_push_at(i, stack, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeDCAAveragePriceViaLen {
fn compute_dca_avg_price_via_len(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
len: usize,
exit: &Exit,
) -> Result<()>;
fn compute_dca_avg_price_via_from(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
from: DateIndex,
exit: &Exit,
) -> Result<()>;
}
impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
fn compute_dca_avg_price_via_len(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ONE + self.inner_version() + stacks.version(),
)?;
let index = max_from.min(DateIndex::from(self.len()));
let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap();
stacks.iter_at(index).try_for_each(|(i, stack)| {
let stack = stack.into_owned();
let mut avg_price = Dollars::from(f64::NAN);
if i > first_price_date {
avg_price = DCA_AMOUNT
* len
.min(i.unwrap_to_usize() + 1)
.min(i.checked_sub(first_price_date).unwrap().unwrap_to_usize() + 1)
/ Bitcoin::from(stack);
}
self.forced_push_at(i, avg_price, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
fn compute_dca_avg_price_via_from(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner_version() + stacks.version(),
)?;
let index = max_from.min(DateIndex::from(self.len()));
let from_usize = from.unwrap_to_usize();
stacks.iter_at(index).try_for_each(|(i, stack)| {
let stack = stack.into_owned();
let mut avg_price = Dollars::from(f64::NAN);
if i >= from {
avg_price =
DCA_AMOUNT * (i.unwrap_to_usize() + 1 - from_usize) / Bitcoin::from(stack);
}
self.forced_push_at(i, avg_price, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeFromSats<I> {
fn compute_from_sats(
&mut self,
max_from: I,
sats: &impl AnyIterableVec<I, Sats>,
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeFromSats<I> for EagerVec<I, Bitcoin>
where
I: StoredIndex,
{
fn compute_from_sats(
&mut self,
max_from: I,
sats: &impl AnyIterableVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner_version() + sats.version(),
)?;
let index = max_from.min(I::from(self.len()));
sats.iter_at(index).try_for_each(|(i, sats)| {
let (i, v) = (i, Bitcoin::from(sats.into_owned()));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeFromBitcoin<I> {
fn compute_from_bitcoin(
&mut self,
max_from: I,
bitcoin: &impl AnyIterableVec<I, Bitcoin>,
price: &impl AnyIterableVec<I, Close<Dollars>>,
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeFromBitcoin<I> for EagerVec<I, Dollars>
where
I: StoredIndex,
{
fn compute_from_bitcoin(
&mut self,
max_from: I,
bitcoin: &impl AnyIterableVec<I, Bitcoin>,
price: &impl AnyIterableVec<I, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner_version() + bitcoin.version(),
)?;
let mut price_iter = price.iter();
let index = max_from.min(I::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| {
let dollars = price_iter.unwrap_get_inner(i);
let (i, v) = (i, *dollars * bitcoin.into_owned());
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeDrawdown<I> {
fn compute_drawdown(
&mut self,
max_from: I,
close: &impl AnyIterableVec<I, Close<Dollars>>,
ath: &impl AnyIterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeDrawdown<I> for EagerVec<I, StoredF32>
where
I: StoredIndex,
{
fn compute_drawdown(
&mut self,
max_from: I,
close: &impl AnyIterableVec<I, Close<Dollars>>,
ath: &impl AnyIterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner_version() + ath.version() + close.version(),
)?;
let index = max_from.min(I::from(self.len()));
let mut close_iter = close.iter();
ath.iter_at(index).try_for_each(|(i, ath)| {
let ath = ath.into_owned();
if ath == Dollars::ZERO {
self.forced_push_at(i, StoredF32::default(), exit)
} else {
let close = *close_iter.unwrap_get_inner(i);
let drawdown = StoredF32::from((*ath - *close) / *ath * -100.0);
self.forced_push_at(i, drawdown, exit)
}
})?;
self.safe_flush(exit)?;
Ok(())
}
}

254
crates/brk_computer/src/transactions.rs
View File

@@ -1,14 +1,15 @@
use std::sync::Arc;
use std::{path::Path, sync::Arc};
use brk_core::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
StoredUsize, TxIndex, TxVersion, Version, Weight,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredBool,
StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight,
};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, File, Format, StoredIndex, VecIterator,
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, StoredIndex,
VecIterator,
};
use crate::grouped::{
@@ -16,89 +17,86 @@ use crate::grouped::{
ComputedVecsFromTxindex, Source, VecBuilderOptions,
};
use super::{Indexes, fetched, indexes};
use super::{Indexes, indexes, price};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
// pub txindex_to_is_v2: LazyVec<Txindex, bool>,
// pub txindex_to_is_v3: LazyVec<Txindex, bool>,
pub indexes_to_coinbase: ComputedValueVecsFromHeight,
pub indexes_to_emptyoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_emptyoutput_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_fee: ComputedValueVecsFromTxindex,
pub indexes_to_feerate: ComputedVecsFromTxindex<Feerate>,
/// Value == 0 when Coinbase
pub txindex_to_input_value: ComputedVecFrom3<
TxIndex,
Sats,
TxIndex,
InputIndex,
TxIndex,
StoredUsize,
InputIndex,
Sats,
>,
pub txindex_to_input_value:
ComputedVecFrom3<TxIndex, Sats, TxIndex, InputIndex, TxIndex, StoredU64, InputIndex, Sats>,
// pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredU64>,
pub txindex_to_output_value: ComputedVecFrom3<
TxIndex,
Sats,
TxIndex,
OutputIndex,
TxIndex,
StoredUsize,
StoredU64,
OutputIndex,
Sats,
>,
// pub indexes_to_output_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_p2a_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2ms_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2pk33_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2pk65_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2pkh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2sh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2tr_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2a_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2ms_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2pk33_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2pk65_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2pkh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2sh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2tr_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_subsidy: ComputedValueVecsFromHeight,
pub indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight,
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredUsize>,
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredU64>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredU64>,
pub inputindex_to_value:
ComputedVecFrom2<InputIndex, Sats, InputIndex, OutputIndex, OutputIndex, Sats>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_is_coinbase: ComputedVecFrom2<TxIndex, bool, TxIndex, Height, Height, TxIndex>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_vsize: ComputedVecFrom1<TxIndex, StoredUsize, TxIndex, Weight>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_is_coinbase:
ComputedVecFrom2<TxIndex, StoredBool, TxIndex, Height, Height, TxIndex>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_vsize: ComputedVecFrom1<TxIndex, StoredU64, TxIndex, Weight>,
pub txindex_to_weight:
ComputedVecFrom2<TxIndex, Weight, TxIndex, StoredU32, TxIndex, StoredU32>,
pub txindex_to_fee: ComputedVecFrom2<TxIndex, Sats, TxIndex, Sats, TxIndex, Sats>,
pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredUsize>,
pub indexes_to_exact_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredU64>,
pub indexes_to_exact_utxo_count: ComputedVecsFromHeight<StoredU64>,
}
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
parent: &Path,
version: Version,
indexer: &Indexer,
indexes: &indexes::Vecs,
computation: Computation,
format: Format,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("transactions"))?);
let compute_dollars = price.is_some();
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
computation,
file,
&file,
"value",
version + VERSION + Version::ZERO,
format,
@@ -125,7 +123,7 @@ impl Vecs {
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
computation,
file,
&file,
"weight",
version + VERSION + Version::ZERO,
format,
@@ -153,7 +151,7 @@ impl Vecs {
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
computation,
file,
&file,
"vsize",
version + VERSION + Version::ZERO,
format,
@@ -161,7 +159,7 @@ impl Vecs {
|index: TxIndex, iter| {
let index = index.unwrap_to_usize();
iter.next_at(index).map(|(_, weight)| {
StoredUsize::from(
StoredU64::from(
bitcoin::Weight::from(weight.into_owned()).to_vbytes_ceil() as usize
)
})
@@ -170,7 +168,7 @@ impl Vecs {
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
computation,
file,
&file,
"is_coinbase",
version + VERSION + Version::ZERO,
format,
@@ -186,15 +184,14 @@ impl Vecs {
.unwrap()
.1
.into_owned();
index == txindex
StoredBool::from(index == txindex)
})
},
)?;
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
computation,
file,
&file,
"input_value",
version + VERSION + Version::ZERO,
format,
@@ -215,7 +212,7 @@ impl Vecs {
.unwrap()
.1
.into_owned();
let range = first_index..first_index + count;
let range = first_index..first_index + count as usize;
range.into_iter().fold(Sats::ZERO, |total, inputindex| {
total
+ inputindex_to_value_iter
@@ -244,7 +241,7 @@ impl Vecs {
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation,
file,
&file,
"output_value",
version + VERSION + Version::ZERO,
format,
@@ -265,7 +262,7 @@ impl Vecs {
.unwrap()
.1
.into_owned();
let range = first_index..first_index + count;
let range = first_index..first_index + count as usize;
range.into_iter().fold(Sats::ZERO, |total, outputindex| {
total
+ outputindex_to_value_iter
@@ -294,7 +291,7 @@ impl Vecs {
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
computation,
file,
&file,
"fee",
version + VERSION + Version::ZERO,
format,
@@ -317,7 +314,7 @@ impl Vecs {
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
computation,
file,
&file,
"feerate",
version + VERSION + Version::ZERO,
format,
@@ -337,7 +334,7 @@ impl Vecs {
Ok(Self {
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"tx_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -352,7 +349,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
file,
&file,
"input_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -367,7 +364,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
file,
&file,
"output_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -382,7 +379,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
file,
&file,
"tx_v1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -392,7 +389,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
file,
&file,
"tx_v2",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -402,7 +399,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
file,
&file,
"tx_v3",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -412,14 +409,14 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
file,
&file,
"fee",
indexes,
Source::Vec(txindex_to_fee.boxed_clone()),
version + VERSION + Version::ZERO,
computation,
format,
fetched,
price,
VecBuilderOptions::default()
.add_sum()
.add_cumulative()
@@ -428,7 +425,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
file,
&file,
"feerate",
Source::None,
version + VERSION + Version::ZERO,
@@ -441,7 +438,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
file,
&file,
"tx_vsize",
Source::None,
version + VERSION + Version::ZERO,
@@ -454,7 +451,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
file,
&file,
"tx_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -467,7 +464,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
file,
&file,
"subsidy",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -483,7 +480,7 @@ impl Vecs {
indexes,
)?,
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
file,
&file,
"coinbase",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -499,7 +496,7 @@ impl Vecs {
indexes,
)?,
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
file,
&file,
"unclaimed_rewards",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -510,7 +507,7 @@ impl Vecs {
indexes,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2a_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -525,7 +522,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2ms_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -540,7 +537,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pk33_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -555,7 +552,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pk65_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -570,7 +567,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2pkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -585,7 +582,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2sh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -600,7 +597,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2tr_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -615,7 +612,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2wpkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -630,7 +627,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"p2wsh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -645,7 +642,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"opreturn_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -660,7 +657,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"unknownoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -675,7 +672,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"emptyoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -690,7 +687,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
file,
&file,
"exact_utxo_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -709,6 +706,8 @@ impl Vecs {
txindex_to_feerate,
txindex_to_vsize,
txindex_to_weight,
file,
})
}
@@ -717,9 +716,9 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.indexes_to_tx_count.compute_all(
indexer,
indexes,
@@ -731,7 +730,8 @@ impl Vecs {
&indexer.vecs.height_to_first_txindex,
&indexer.vecs.txindex_to_txid,
exit,
)
)?;
Ok(())
},
)?;
@@ -752,7 +752,7 @@ impl Vecs {
)?;
let compute_indexes_to_tx_vany =
|indexes_to_tx_vany: &mut ComputedVecsFromHeight<StoredUsize>, txversion| {
|indexes_to_tx_vany: &mut ComputedVecsFromHeight<StoredU64>, txversion| {
let mut txindex_to_txversion_iter = indexer.vecs.txindex_to_txversion.iter();
indexes_to_tx_vany.compute_all(
indexer,
@@ -769,7 +769,8 @@ impl Vecs {
v == txversion
},
exit,
)
)?;
Ok(())
},
)
};
@@ -863,7 +864,7 @@ impl Vecs {
starting_indexes,
exit,
Some(&self.txindex_to_fee),
fetched,
price,
)?;
self.indexes_to_feerate.compute_rest(
@@ -893,7 +894,7 @@ impl Vecs {
self.indexes_to_coinbase.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, indexer, _, starting_indexes, exit| {
@@ -910,23 +911,23 @@ impl Vecs {
.unwrap_to_usize();
let output_count = txindex_to_output_count_iter.unwrap_get_inner(txindex);
let mut sats = Sats::ZERO;
(first_outputindex..first_outputindex + *output_count).for_each(
|outputindex| {
(first_outputindex..first_outputindex + usize::from(output_count))
.for_each(|outputindex| {
sats += outputindex_to_value_iter
.unwrap_get_inner(OutputIndex::from(outputindex));
},
);
});
(height, sats)
},
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_subsidy.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -940,14 +941,15 @@ impl Vecs {
(height, coinbase.checked_sub(fees).unwrap())
},
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_unclaimed_rewards.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -961,7 +963,8 @@ impl Vecs {
(height, expected.checked_sub(subsidy).unwrap())
},
exit,
)
)?;
Ok(())
},
)?;
@@ -976,7 +979,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2aaddressindex,
&indexer.vecs.p2aaddressindex_to_p2abytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -991,7 +995,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2msoutputindex,
&indexer.vecs.p2msoutputindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1006,7 +1011,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2pk33addressindex,
&indexer.vecs.p2pk33addressindex_to_p2pk33bytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1021,7 +1027,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2pk65addressindex,
&indexer.vecs.p2pk65addressindex_to_p2pk65bytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1036,7 +1043,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2pkhaddressindex,
&indexer.vecs.p2pkhaddressindex_to_p2pkhbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1051,7 +1059,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2shaddressindex,
&indexer.vecs.p2shaddressindex_to_p2shbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1066,7 +1075,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2traddressindex,
&indexer.vecs.p2traddressindex_to_p2trbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1081,7 +1091,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2wpkhaddressindex,
&indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1096,7 +1107,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2wshaddressindex,
&indexer.vecs.p2wshaddressindex_to_p2wshbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1111,7 +1123,8 @@ impl Vecs {
&indexer.vecs.height_to_first_opreturnindex,
&indexer.vecs.opreturnindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1126,7 +1139,8 @@ impl Vecs {
&indexer.vecs.height_to_first_unknownoutputindex,
&indexer.vecs.unknownoutputindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1141,7 +1155,8 @@ impl Vecs {
&indexer.vecs.height_to_first_emptyoutputindex,
&indexer.vecs.emptyoutputindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1167,7 +1182,7 @@ impl Vecs {
|(h, output_count, ..)| {
let input_count = input_count_iter.unwrap_get_inner(h);
let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);
let block_count = usize::from(h + 1_usize);
let block_count = u64::from(h + 1_usize);
// -1 > genesis output is unspendable
let mut utxo_count =
*output_count - (*input_count - block_count) - *opreturn_count - 1;
@@ -1190,13 +1205,16 @@ impl Vecs {
utxo_count -= 1;
}
(h, StoredUsize::from(utxo_count))
(h, StoredU64::from(utxo_count))
},
exit,
)
)?;
Ok(())
},
)?;
self.file.flush()?;
self.file.punch_holes()?;
Ok(())
}

89
crates/brk_core/src/structs/_addressindex.rs
View File

@@ -1,89 +0,0 @@
use std::ops::Add;
use byteview::ByteView;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct AddressIndex(u32);
impl AddressIndex {
pub const BYTES: usize = size_of::<Self>();
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(self.0 + 1)
}
}
impl From<u32> for AddressIndex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for AddressIndex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<AddressIndex> for u64 {
fn from(value: AddressIndex) -> Self {
value.0 as u64
}
}
impl From<usize> for AddressIndex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<AddressIndex> for usize {
fn from(value: AddressIndex) -> Self {
value.0 as usize
}
}
impl From<ByteView> for AddressIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<AddressIndex> for ByteView {
fn from(value: AddressIndex) -> Self {
Self::new(value.as_bytes())
}
}
impl Add<usize> for AddressIndex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u32)
}
}
impl Add<AddressIndex> for AddressIndex {
type Output = Self;
fn add(self, rhs: AddressIndex) -> Self::Output {
Self(self.0 + rhs.0)
}
}

27
crates/brk_core/src/structs/_addressindexoutputindex.rs
View File

@@ -1,27 +0,0 @@
use byteview::ByteView;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
use super::{AddressIndex, Outputindex};
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes,
)]
#[repr(C)]
pub struct AddressIndexOutputIndex {
addressindex: AddressIndex,
_padding: u32,
outputindex: Outputindex,
}
impl From<ByteView> for AddressIndexOutputIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<AddressIndexOutputIndex> for ByteView {
fn from(value: AddressIndexOutputIndex) -> Self {
Self::new(value.as_bytes())
}
}

48
crates/brk_core/src/structs/_addresstype.rs
View File

@@ -1,48 +0,0 @@
use serde::Serialize;
use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
use super::OutputType;
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
TryFromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
#[repr(u8)]
pub enum AddressType {
P2PK65,
P2PK33,
P2PKH,
P2SH,
P2WPKH,
P2WSH,
P2TR,
P2A,
}
impl From<OutputType> for AddressType {
fn from(value: OutputType) -> Self {
match value {
OutputType::P2A => Self::P2A,
OutputType::P2PK33 => Self::P2PK33,
OutputType::P2PK65 => Self::P2PK65,
OutputType::P2PKH => Self::P2PKH,
OutputType::P2SH => Self::P2SH,
OutputType::P2TR => Self::P2TR,
OutputType::P2WPKH => Self::P2WPKH,
OutputType::P2WSH => Self::P2WSH,
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
}
}
}

135
crates/brk_core/src/structs/outputtype.rs
View File

@@ -1,135 +0,0 @@
use bitcoin::{ScriptBuf, opcodes::all::OP_PUSHBYTES_2};
use serde::Serialize;
use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
TryFromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
#[repr(u8)]
pub enum OutputType {
P2PK65,
P2PK33,
P2PKH,
P2MS,
P2SH,
OpReturn,
P2WPKH,
P2WSH,
P2TR,
P2A,
Empty = 254,
Unknown = 255,
}
impl OutputType {
pub fn is_spendable(&self) -> bool {
match self {
Self::P2PK65 => true,
Self::P2PK33 => true,
Self::P2PKH => true,
Self::P2MS => true,
Self::P2SH => true,
Self::OpReturn => false,
Self::P2WPKH => true,
Self::P2WSH => true,
Self::P2TR => true,
Self::P2A => true,
Self::Empty => true,
Self::Unknown => true,
}
}
pub fn is_address(&self) -> bool {
match self {
Self::P2PK65 => true,
Self::P2PK33 => true,
Self::P2PKH => true,
Self::P2MS => false,
Self::P2SH => true,
Self::OpReturn => false,
Self::P2WPKH => true,
Self::P2WSH => true,
Self::P2TR => true,
Self::P2A => true,
Self::Empty => false,
Self::Unknown => false,
}
}
pub fn is_not_address(&self) -> bool {
!self.is_address()
}
pub fn is_unspendable(&self) -> bool {
!self.is_spendable()
}
pub fn as_vec() -> Vec<Self> {
vec![
Self::P2PK65,
Self::P2PK33,
Self::P2PKH,
Self::P2MS,
Self::P2SH,
Self::OpReturn,
Self::P2WPKH,
Self::P2WSH,
Self::P2TR,
Self::P2A,
Self::Empty,
Self::Unknown,
]
}
}
impl From<&ScriptBuf> for OutputType {
fn from(script: &ScriptBuf) -> Self {
if script.is_p2pk() {
let bytes = script.as_bytes();
match bytes.len() {
67 => Self::P2PK65,
35 => Self::P2PK33,
_ => {
dbg!(bytes);
unreachable!()
}
}
} else if script.is_p2pkh() {
Self::P2PKH
} else if script.is_multisig() {
Self::P2MS
} else if script.is_p2sh() {
Self::P2SH
} else if script.is_op_return() {
Self::OpReturn
} else if script.is_p2wpkh() {
Self::P2WPKH
} else if script.is_p2wsh() {
Self::P2WSH
} else if script.is_p2tr() {
Self::P2TR
} else if script.witness_version() == Some(bitcoin::WitnessVersion::V1)
&& script.len() == 4
&& script.as_bytes()[1] == OP_PUSHBYTES_2.to_u8()
&& script.as_bytes()[2..4] == [78, 115]
{
Self::P2A
} else if script.is_empty() {
Self::Empty
} else {
Self::Unknown
}
}
}

115
crates/brk_core/src/structs/stored_u64.rs
View File

@@ -1,115 +0,0 @@
use std::ops::{Add, AddAssign, Div};
use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Printable};
use super::{InputIndex, OutputIndex, TxIndex};
#[derive(
Debug,
Deref,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct StoredU64(u64);
impl StoredU64 {
pub const ZERO: Self = Self(0);
pub fn new(counter: u64) -> Self {
Self(counter)
}
}
impl From<u64> for StoredU64 {
fn from(value: u64) -> Self {
Self(value)
}
}
impl From<usize> for StoredU64 {
fn from(value: usize) -> Self {
Self(value as u64)
}
}
impl CheckedSub<StoredU64> for StoredU64 {
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self)
}
}
impl Div<usize> for StoredU64 {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
Self(self.0 / rhs as u64)
}
}
impl Add for StoredU64 {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl AddAssign for StoredU64 {
fn add_assign(&mut self, rhs: Self) {
*self = *self + rhs
}
}
impl From<f64> for StoredU64 {
fn from(value: f64) -> Self {
if value < 0.0 || value > u32::MAX as f64 {
panic!()
}
Self(value as u64)
}
}
impl From<StoredU64> for f64 {
fn from(value: StoredU64) -> Self {
value.0 as f64
}
}
impl From<TxIndex> for StoredU64 {
fn from(value: TxIndex) -> Self {
Self(*value as u64)
}
}
impl From<InputIndex> for StoredU64 {
fn from(value: InputIndex) -> Self {
Self(*value)
}
}
impl From<OutputIndex> for StoredU64 {
fn from(value: OutputIndex) -> Self {
Self(*value)
}
}
impl Printable for StoredU64 {
fn to_string() -> &'static str {
"u64"
}
fn to_possible_strings() -> &'static [&'static str] {
&["u64"]
}
}

5
crates/brk_core/src/traits/mod.rs
View File

@@ -1,5 +0,0 @@
mod from_coarser;
mod printable;
pub use from_coarser::*;
pub use printable::*;

4
crates/brk_core/src/traits/printable.rs
View File

@@ -1,4 +0,0 @@
pub trait Printable {
fn to_string() -> &'static str;
fn to_possible_strings() -> &'static [&'static str];
}

27
crates/brk_core/src/utils/bytes.rs
View File

@@ -1,27 +0,0 @@
use crate::{Error, Result};
#[allow(clippy::result_unit_err)]
pub fn copy_first_4bytes(slice: &[u8]) -> Result<[u8; 4]> {
let mut buf: [u8; 4] = [0; 4];
let buf_len = buf.len();
if slice.len() < buf_len {
return Err(Error::Str("Buffer is too small to convert to 8 bytes"));
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(buf)
}
#[allow(clippy::result_unit_err)]
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
return Err(Error::Str("Buffer is too small to convert to 8 bytes"));
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(buf)
}

13
crates/brk_core/src/utils/mod.rs
View File

@@ -1,13 +0,0 @@
mod bytes;
mod checked_sub;
mod paths;
mod pause;
mod rlimit;
mod serde;
pub use bytes::*;
pub use checked_sub::*;
pub use paths::*;
pub use pause::*;
pub use rlimit::*;
pub use serde::*;

9
crates/brk_core/src/utils/pause.rs
View File

@@ -1,9 +0,0 @@
use log::info;
pub fn pause() {
info!("Press enter to continue...");
let mut buffer = String::new();
std::io::stdin()
.read_line(&mut buffer)
.expect("Failed to read line");
}

15
crates/brk_core/src/utils/rlimit.rs
View File

@@ -1,15 +0,0 @@
use std::io;
use rlimit::{Resource, getrlimit};
pub fn setrlimit() -> io::Result<()> {
let no_file_limit = getrlimit(Resource::NOFILE)?;
rlimit::setrlimit(
Resource::NOFILE,
no_file_limit.0.max(10_000),
no_file_limit.1,
)?;
Ok(())
}

12
crates/brk_core/src/utils/serde.rs
View File

@@ -1,12 +0,0 @@
use serde::{Deserialize, Deserializer};
pub fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: Deserialize<'de> + Default,
{
match T::deserialize(deserializer) {
Ok(v) => Ok(v),
Err(_) => Ok(T::default()),
}
}

16
crates/brk_error/Cargo.toml Normal file
View File

@@ -0,0 +1,16 @@
[package]
name = "brk_error"
description = "Errors used throughout the Bitcoin Research Kit"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
brk_vecs = { workspace = true }
bitcoincore-rpc = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
minreq = { workspace = true }
zerocopy = { workspace = true }

20
crates/brk_core/README.md → crates/brk_error/README.md
View File

@@ -7,15 +7,15 @@
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
<a href="https://crates.io/crates/brk_core">
<img src="https://img.shields.io/crates/v/brk_core" alt="Version" />
<a href="https://crates.io/crates/brk_structs">
<img src="https://img.shields.io/crates/v/brk_structs" alt="Version" />
</a>
<a href="https://docs.rs/brk_core">
<img src="https://img.shields.io/docsrs/brk_core" alt="Documentation" />
<a href="https://docs.rs/brk_structs">
<img src="https://img.shields.io/docsrs/brk_structs" alt="Documentation" />
</a>
<img src="https://img.shields.io/crates/size/brk_core" alt="Size" />
<a href="https://deps.rs/crate/brk_core">
<img src="https://deps.rs/crate/brk_core/latest/status.svg" alt="Dependency status">
<img src="https://img.shields.io/crates/size/brk_structs" alt="Size" />
<a href="https://deps.rs/crate/brk_structs">
<img src="https://deps.rs/crate/brk_structs/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/HaR3wpH3nr">
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A list of structs that are used throughout the project as units, think of `Date`, `Height`, `Sats`, `Txindex` or anything that can be either a key and/or a value of a dataset.

105
crates/brk_error/src/lib.rs Normal file
View File

@@ -0,0 +1,105 @@
use std::{
fmt::{self, Debug},
io, result, time,
};
pub type Result<T, E = Error> = result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
IO(io::Error),
BitcoinRPC(bitcoincore_rpc::Error),
Jiff(jiff::Error),
Fjall(fjall::Error),
Minreq(minreq::Error),
SystemTimeError(time::SystemTimeError),
ZeroCopyError,
Vecs(brk_vecs::Error),
WrongLength,
WrongAddressType,
UnindexableDate,
Str(&'static str),
String(String),
}
impl From<time::SystemTimeError> for Error {
fn from(value: time::SystemTimeError) -> Self {
Self::SystemTimeError(value)
}
}
impl From<io::Error> for Error {
fn from(value: io::Error) -> Self {
Self::IO(value)
}
}
impl From<bitcoincore_rpc::Error> for Error {
fn from(value: bitcoincore_rpc::Error) -> Self {
Self::BitcoinRPC(value)
}
}
impl From<minreq::Error> for Error {
fn from(value: minreq::Error) -> Self {
Self::Minreq(value)
}
}
impl From<brk_vecs::Error> for Error {
fn from(value: brk_vecs::Error) -> Self {
Self::Vecs(value)
}
}
impl From<jiff::Error> for Error {
fn from(value: jiff::Error) -> Self {
Self::Jiff(value)
}
}
impl From<fjall::Error> for Error {
fn from(value: fjall::Error) -> Self {
Self::Fjall(value)
}
}
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
}
}
impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
Self::ZeroCopyError
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::IO(error) => Debug::fmt(&error, f),
Error::Minreq(error) => Debug::fmt(&error, f),
Error::Vecs(error) => Debug::fmt(&error, f),
Error::BitcoinRPC(error) => Debug::fmt(&error, f),
Error::SystemTimeError(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::Fjall(error) => Debug::fmt(&error, f),
Error::ZeroCopyError => write!(f, "ZeroCopy error"),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(
f,
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
),
Error::Str(s) => write!(f, "{s}"),
Error::String(s) => write!(f, "{s}"),
}
}
}
impl std::error::Error for Error {}

14
crates/brk_exit/Cargo.toml
View File

@@ -1,14 +0,0 @@
[package]
name = "brk_exit"
description = "An exit blocker built on top of ctrlc"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
brk_logger = { workspace = true }
ctrlc = { version = "3.4.7", features = ["termination"] }
log = { workspace = true }
parking_lot = { workspace = true }

36
crates/brk_exit/README.md
View File

@@ -1,36 +0,0 @@
# BRK Exit
<p align="left">
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
<a href="https://crates.io/crates/brk_exit">
<img src="https://img.shields.io/crates/v/brk_exit" alt="Version" />
</a>
<a href="https://docs.rs/brk_exit">
<img src="https://img.shields.io/docsrs/brk_exit" alt="Documentation" />
</a>
<img src="https://img.shields.io/crates/size/brk_exit" alt="Size" />
<a href="https://deps.rs/crate/brk_exit">
<img src="https://deps.rs/crate/brk_exit/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/HaR3wpH3nr">
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A simple crate that stops the program from exitting when blocking is activated until it is released. The purpose of that is to prevent exitting when a program is in the middle of saving data and thus prevent partial writes.
It's built on top of [ctrlc](https://crates.io/crates/ctrlc) which handles Ctrl + C (SIGINT), stopping the program using the `kill` command (SIGTERM) and closing the terminal (SIGHUP) but it doesn't support force kills (`kill -9`).

28
crates/brk_exit/examples/main.rs
View File

@@ -1,28 +0,0 @@
use std::{path::Path, thread::sleep, time::Duration};
use brk_exit::Exit;
use log::info;
fn main() {
let exit = Exit::new();
brk_logger::init(Some(Path::new(".log")));
let lock = exit.lock();
let mut i = 0;
while i < 21 {
info!("i = {i}");
sleep(Duration::from_secs(1));
i += 1;
}
drop(lock);
let mut j = 0;
while j < 10 {
info!("j = {j}");
sleep(Duration::from_secs(1));
j += 1;
}
}

4
crates/brk_fetcher/Cargo.toml
View File

@@ -8,9 +8,9 @@ homepage.workspace = true
repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_error = { workspace = true }
brk_logger = { workspace = true }
color-eyre = { workspace = true }
brk_structs = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
serde_json = { workspace = true }

6
crates/brk_fetcher/README.md
View File

@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A crate that can fetch the Bitcoin price, either by date or height, from Binance, Kraken and the main instance of BRK.

7
crates/brk_fetcher/examples/main.rs
View File

@@ -1,9 +1,8 @@
use brk_core::{Date, Height};
use brk_error::Result;
use brk_fetcher::{BRK, Binance, Fetcher, Kraken};
use brk_structs::{Date, Height};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
fn main() -> Result<()> {
brk_logger::init(None);
let mut brk = BRK::default();

46
crates/brk_fetcher/src/binance.rs
View File

@@ -6,8 +6,8 @@ use std::{
str::FromStr,
};
use brk_core::{Cents, OHLCCents, Timestamp};
use color_eyre::eyre::{ContextCompat, eyre};
use brk_error::{Error, Result};
use brk_structs::{Cents, OHLCCents, Timestamp};
use log::info;
use serde_json::Value;
@@ -35,7 +35,7 @@ impl Binance {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
@@ -65,7 +65,7 @@ impl Binance {
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Binance...");
retry(
@@ -81,7 +81,7 @@ impl Binance {
)
}
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLCCents> {
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date {
self._1d.replace(Self::fetch_1d()?);
}
@@ -91,10 +91,10 @@ impl Binance {
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
.ok_or(Error::Str("Couldn't find date"))
}
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Binance...");
retry(
@@ -104,9 +104,9 @@ impl Binance {
)
}
fn read_har(&self) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn read_har(&self) -> Result<BTreeMap<Timestamp, OHLCCents>> {
if self.path.is_none() {
return Err(eyre!("Path missing"));
return Err(Error::Str("Path missing"));
}
info!("Reading Binance har file...");
@@ -120,7 +120,7 @@ impl Binance {
let file = if let Ok(file) = File::open(path_binance_har) {
file
} else {
return Err(eyre!("Missing binance file"));
return Err(Error::Str("Missing binance file"));
};
let reader = BufReader::new(file);
@@ -132,13 +132,13 @@ impl Binance {
};
json.get("log")
.context("Expect object to have log attribute")?
.ok_or(Error::Str("Expect object to have log attribute"))?
.as_object()
.context("Expect to be an object")?
.ok_or(Error::Str("Expect to be an object"))?
.get("entries")
.context("Expect object to have entries")?
.ok_or(Error::Str("Expect object to have entries"))?
.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.filter(|entry| {
entry
@@ -181,30 +181,28 @@ impl Binance {
})
}
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
fn json_to_date_to_ohlc(json: &Value) -> Result<BTreeMap<Date, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
}
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> Result<BTreeMap<K, V>>
where
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
F: Fn(&Value) -> Result<(K, V)>,
K: Ord,
{
json.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(fun)
.collect::<Result<BTreeMap<_, _>, _>>()
}
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> {
let array = array.as_array().context("Expect to be array")?;
fn array_to_timestamp_and_ohlc(array: &Value) -> Result<(Timestamp, OHLCCents)> {
let array = array.as_array().ok_or(Error::Str("Expect to be array"))?;
let timestamp = Timestamp::from((array.first().unwrap().as_u64().unwrap() / 1_000) as u32);
@@ -231,7 +229,7 @@ impl Binance {
))
}
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> {
fn array_to_date_and_ohlc(array: &Value) -> Result<(Date, OHLCCents)> {
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
}

32
crates/brk_fetcher/src/brk.rs
View File

@@ -1,7 +1,7 @@
use std::collections::BTreeMap;
use brk_core::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
use color_eyre::eyre::{ContextCompat, eyre};
use brk_error::{Error, Result};
use brk_structs::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
use log::info;
use serde_json::Value;
@@ -19,7 +19,7 @@ const RETRIES: usize = 10;
const CHUNK_SIZE: usize = 10_000;
impl BRK {
pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
pub fn get_from_height(&mut self, height: Height) -> Result<OHLCCents> {
let key = height.checked_sub(height % CHUNK_SIZE).unwrap();
#[allow(clippy::map_entry)]
@@ -39,10 +39,10 @@ impl BRK {
.unwrap()
.get(usize::from(height.checked_sub(key).unwrap()))
.cloned()
.ok_or(eyre!("Couldn't find height in BRK"))
.ok_or(Error::Str("Couldn't find height in BRK"))
}
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
fn fetch_height_prices(height: Height) -> Result<Vec<OHLCCents>> {
info!("Fetching BRK height {height} prices...");
retry(
@@ -56,7 +56,7 @@ impl BRK {
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
@@ -66,7 +66,7 @@ impl BRK {
)
}
pub fn get_from_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
pub fn get_from_date(&mut self, date: Date) -> Result<OHLCCents> {
let dateindex = DateIndex::try_from(date)?;
let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap();
@@ -88,10 +88,10 @@ impl BRK {
.unwrap()
.get(usize::from(dateindex.checked_sub(key).unwrap()))
.cloned()
.ok_or(eyre!("Couldn't find date in BRK"))
.ok_or(Error::Str("Couldn't find date in BRK"))
}
fn fetch_date_prices(dateindex: DateIndex) -> color_eyre::Result<Vec<OHLCCents>> {
fn fetch_date_prices(dateindex: DateIndex) -> Result<Vec<OHLCCents>> {
info!("Fetching BRK dateindex {dateindex} prices...");
retry(
@@ -105,7 +105,7 @@ impl BRK {
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
@@ -115,15 +115,17 @@ impl BRK {
)
}
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
let ohlc = value.as_array().context("Expect as_array to work")?;
fn value_to_ohlc(value: &Value) -> Result<OHLCCents> {
let ohlc = value
.as_array()
.ok_or(Error::Str("Expect as_array to work"))?;
let get_value = |index: usize| -> color_eyre::Result<_> {
let get_value = |index: usize| -> Result<_> {
Ok(Cents::from(Dollars::from(
ohlc.get(index)
.context("Expect index key to work")?
.ok_or(Error::Str("Expect index key to work"))?
.as_f64()
.context("Expect as_f64 to work")?,
.ok_or(Error::Str("Expect as_f64 to work"))?,
)))
};

40
crates/brk_fetcher/src/kraken.rs
View File

@@ -1,7 +1,7 @@
use std::collections::BTreeMap;
use brk_core::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp};
use color_eyre::eyre::ContextCompat;
use brk_error::{Error, Result};
use brk_structs::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp};
use log::info;
use serde_json::Value;
@@ -18,7 +18,7 @@ impl Kraken {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
@@ -32,7 +32,7 @@ impl Kraken {
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Kraken...");
retry(
@@ -42,7 +42,7 @@ impl Kraken {
)
}
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLCCents> {
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date {
self._1d.replace(Kraken::fetch_1d()?);
}
@@ -51,10 +51,10 @@ impl Kraken {
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
.ok_or(Error::Str("Couldn't find date"))
}
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Kraken...");
retry(
@@ -64,38 +64,36 @@ impl Kraken {
)
}
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
fn json_to_date_to_ohlc(json: &Value) -> Result<BTreeMap<Date, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
}
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> Result<BTreeMap<K, V>>
where
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
F: Fn(&Value) -> Result<(K, V)>,
K: Ord,
{
json.as_object()
.context("Expect to be an object")?
.ok_or(Error::Str("Expect to be an object"))?
.get("result")
.context("Expect object to have result")?
.ok_or(Error::Str("Expect object to have result"))?
.as_object()
.context("Expect to be an object")?
.ok_or(Error::Str("Expect to be an object"))?
.get("XXBTZUSD")
.context("Expect to have XXBTZUSD")?
.ok_or(Error::Str("Expect to have XXBTZUSD"))?
.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(fun)
.collect::<Result<BTreeMap<_, _>, _>>()
}
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> {
let array = array.as_array().context("Expect to be array")?;
fn array_to_timestamp_and_ohlc(array: &Value) -> Result<(Timestamp, OHLCCents)> {
let array = array.as_array().ok_or(Error::Str("Expect to be array"))?;
let timestamp = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32);
@@ -122,7 +120,7 @@ impl Kraken {
))
}
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> {
fn array_to_date_and_ohlc(array: &Value) -> Result<(Date, OHLCCents)> {
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
}

18
crates/brk_fetcher/src/lib.rs
View File

@@ -5,8 +5,8 @@
use std::{collections::BTreeMap, path::Path, thread::sleep, time::Duration};
use brk_core::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
use color_eyre::eyre::Error;
use brk_error::{Error, Result};
use brk_structs::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
use log::info;
mod binance;
@@ -29,7 +29,7 @@ pub struct Fetcher {
}
impl Fetcher {
pub fn import(hars_path: Option<&Path>) -> color_eyre::Result<Self> {
pub fn import(hars_path: Option<&Path>) -> Result<Self> {
Ok(Self {
binance: Binance::init(hars_path),
kraken: Kraken::default(),
@@ -37,11 +37,11 @@ impl Fetcher {
})
}
pub fn get_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
pub fn get_date(&mut self, date: Date) -> Result<OHLCCents> {
self.get_date_(date, 0)
}
fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
fn get_date_(&mut self, date: Date, tries: usize) -> Result<OHLCCents> {
self.kraken
.get_from_1d(&date)
.or_else(|_| {
@@ -72,7 +72,7 @@ impl Fetcher {
height: Height,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
self.get_height_(height, timestamp, previous_timestamp, 0)
}
@@ -82,7 +82,7 @@ impl Fetcher {
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
tries: usize,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
let timestamp = timestamp.floor_seconds();
if previous_timestamp.is_none() && height != Height::ZERO {
@@ -149,7 +149,7 @@ How to fix this:
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
name: &str,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
let previous_ohlc = previous_timestamp
.map_or(Some(OHLCCents::default()), |previous_timestamp| {
tree.get(&previous_timestamp).cloned()
@@ -158,7 +158,7 @@ How to fix this:
let last_ohlc = tree.get(&timestamp);
if previous_ohlc.is_none() || last_ohlc.is_none() {
return Err(Error::msg(format!("Couldn't find timestamp in {name}")));
return Err(Error::String(format!("Couldn't find timestamp in {name}")));
}
let previous_ohlc = previous_ohlc.unwrap();

7
crates/brk_fetcher/src/retry.rs
View File

@@ -1,12 +1,9 @@
use std::{fmt::Debug, thread::sleep, time::Duration};
use brk_error::Result;
use log::info;
pub fn retry<T>(
function: impl Fn(usize) -> color_eyre::Result<T>,
sleep_in_s: u64,
retries: usize,
) -> color_eyre::Result<T>
pub fn retry<T>(function: impl Fn(usize) -> Result<T>, sleep_in_s: u64, retries: usize) -> Result<T>
where
T: Debug,
{

5
crates/brk_indexer/Cargo.toml
View File

@@ -10,13 +10,12 @@ repository.workspace = true
[dependencies]
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_structs = { workspace = true }
brk_error = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vecs = { workspace = true }
color-eyre = { workspace = true }
fjall = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }

6
crates/brk_indexer/README.md
View File

@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../brk_parser`) and creates a database of the vecs (`brk_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code.

14
crates/brk_indexer/examples/indexer.rs
View File

@@ -5,20 +5,19 @@ use std::{
time::{Duration, Instant},
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vecs::Exit;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));
let bitcoin_dir = brk_core::default_bitcoin_path();
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
// let bitcoin_dir = brk_structs::default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let outputs_dir = Path::new("./_outputs");
fs::create_dir_all(outputs_dir)?;
// let outputs_dir = brk_core::default_brk_path().join("outputs");
// let outputs_dir = brk_structs::default_brk_path().join("outputs");
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
@@ -26,6 +25,7 @@ fn main() -> color_eyre::Result<()> {
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);

55
crates/brk_indexer/src/indexes.rs
View File

@@ -1,13 +1,15 @@
use bitcoincore_rpc::Client;
use brk_core::{
use brk_error::{Error, Result};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_structs::{
BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex,
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
Result, TxIndex, TypeIndex, UnknownOutputIndex,
TxIndex, TypeIndex, UnknownOutputIndex,
};
use brk_vecs::{
AnyIterableVec, AnyStoredIterableVec, GenericStoredVec, StoredIndex, StoredRaw, VecIterator,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vecs::{AnyIterableVec, AnyStampedVec, AnyVec, StampedVec, StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::{Stores, Vecs};
@@ -46,6 +48,7 @@ impl Indexes {
OutputType::P2WPKH => *self.p2wpkhaddressindex,
OutputType::P2WSH => *self.p2wshaddressindex,
OutputType::Unknown => *self.unknownoutputindex,
_ => unreachable!(),
}
}
@@ -87,8 +90,8 @@ impl Indexes {
}
impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
type Error = color_eyre::Report;
fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result<Self> {
type Error = Error;
fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> Result<Self> {
// Height at which we want to start: min last saved + 1 or 0
let vecs_starting_height = vecs.starting_height();
let stores_starting_height = stores.starting_height();
@@ -123,100 +126,100 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
&vecs.emptyoutputindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
height,
p2msoutputindex: starting_index(
&vecs.height_to_first_p2msoutputindex,
&vecs.p2msoutputindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
opreturnindex: starting_index(
&vecs.height_to_first_opreturnindex,
&vecs.opreturnindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2pk33addressindex: starting_index(
&vecs.height_to_first_p2pk33addressindex,
&vecs.p2pk33addressindex_to_p2pk33bytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2pk65addressindex: starting_index(
&vecs.height_to_first_p2pk65addressindex,
&vecs.p2pk65addressindex_to_p2pk65bytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2pkhaddressindex: starting_index(
&vecs.height_to_first_p2pkhaddressindex,
&vecs.p2pkhaddressindex_to_p2pkhbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2shaddressindex: starting_index(
&vecs.height_to_first_p2shaddressindex,
&vecs.p2shaddressindex_to_p2shbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2traddressindex: starting_index(
&vecs.height_to_first_p2traddressindex,
&vecs.p2traddressindex_to_p2trbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2wpkhaddressindex: starting_index(
&vecs.height_to_first_p2wpkhaddressindex,
&vecs.p2wpkhaddressindex_to_p2wpkhbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2wshaddressindex: starting_index(
&vecs.height_to_first_p2wshaddressindex,
&vecs.p2wshaddressindex_to_p2wshbytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
p2aaddressindex: starting_index(
&vecs.height_to_first_p2aaddressindex,
&vecs.p2aaddressindex_to_p2abytes,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
txindex: starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)
.context("")?,
.ok_or(Error::Str(""))?,
inputindex: starting_index(
&vecs.height_to_first_inputindex,
&vecs.inputindex_to_outputindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
outputindex: starting_index(
&vecs.height_to_first_outputindex,
&vecs.outputindex_to_value,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
unknownoutputindex: starting_index(
&vecs.height_to_first_unknownoutputindex,
&vecs.unknownoutputindex_to_txindex,
height,
)
.context("")?,
.ok_or(Error::Str(""))?,
})
}
}
pub fn starting_index<I, T>(
height_to_index: &StampedVec<Height, I>,
index_to_else: &StampedVec<I, T>,
height_to_index: &impl AnyStoredIterableVec<Height, I>,
index_to_else: &impl AnyIterableVec<I, T>,
starting_height: Height,
) -> Option<I>
where
I: StoredType + StoredIndex + From<usize>,
T: StoredType,
I: StoredRaw + StoredIndex + From<usize>,
T: StoredRaw,
{
let h = Height::from(u64::from(height_to_index.stamp()));
if h.is_zero() {

88
crates/brk_indexer/src/lib.rs
View File

@@ -6,16 +6,16 @@
use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant};
use bitcoin::{Transaction, TxIn, TxOut};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Result, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit,
};
use brk_exit::Exit;
use brk_error::{Error, Result};
use brk_parser::Parser;
use brk_store::AnyStore;
use brk_vecs::{AnyVec, File, PAGE_SIZE, Reader, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use brk_structs::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Sats, StoredBool, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout,
};
use brk_vecs::{AnyVec, Exit, File, GenericStoredVec, PAGE_SIZE, Reader, VecIterator};
use log::{error, info};
use rayon::prelude::*;
mod indexes;
@@ -38,9 +38,7 @@ pub struct Indexer {
}
impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result<Self> {
setrlimit()?;
pub fn forced_import(outputs_dir: &Path) -> Result<Self> {
let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?);
let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?;
@@ -63,7 +61,7 @@ impl Indexer {
rpc: &'static bitcoincore_rpc::Client,
exit: &Exit,
check_collisions: bool,
) -> color_eyre::Result<Indexes> {
) -> Result<Indexes> {
let file = self.file.clone();
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc))
@@ -177,7 +175,7 @@ impl Indexer {
);
parser.parse(start, end).iter().try_for_each(
|(height, block, blockhash)| -> color_eyre::Result<()> {
|(height, block, blockhash)| -> Result<()> {
info!("Indexing block {height}...");
idxs.height = height;
@@ -204,7 +202,7 @@ impl Indexer {
.is_some_and(|prev_height| *prev_height != height)
{
error!("BlockHash: {blockhash}");
return Err(eyre!("Collision, expect prefix to need be set yet"));
return Err(Error::Str("Collision, expect prefix to need be set yet"));
}
idxs.push_if_needed(vecs)?;
@@ -227,7 +225,7 @@ impl Indexer {
outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle,
) = thread::scope(|scope| {
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle =
scope.spawn(|| -> color_eyre::Result<_> {
scope.spawn(|| -> Result<_> {
block
.txdata
.iter()
@@ -247,7 +245,7 @@ impl Indexer {
Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt)))
})
.collect::<color_eyre::Result<BTreeMap<_, _>>>()
.collect::<Result<BTreeMap<_, _>>>()
});
let input_source_vec_handle = scope.spawn(|| {
@@ -266,7 +264,7 @@ impl Indexer {
inputs
.into_par_iter()
.enumerate()
.map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> color_eyre::Result<(InputIndex, InputSource)> {
.map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> Result<(InputIndex, InputSource)> {
let txindex = idxs.txindex + block_txindex;
let inputindex = idxs.inputindex + InputIndex::from(block_inputindex);
@@ -294,7 +292,7 @@ impl Indexer {
let vout = Vout::from(outpoint.vout);
let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_mmap)?
.context("Expect outputindex to not be none")
.ok_or(Error::Str("Expect outputindex to not be none"))
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
})?.into_owned()
@@ -306,7 +304,7 @@ impl Indexer {
outputindex,
))))
})
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
.try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
@@ -337,14 +335,14 @@ impl Indexer {
.enumerate()
.map(
#[allow(clippy::type_complexity)]
|(block_outputindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<(
|(block_outputindex, (block_txindex, vout, txout, tx))| -> Result<(
OutputIndex,
(
&TxOut,
TxIndex,
Vout,
OutputType,
brk_core::Result<AddressBytes>,
Result<AddressBytes>,
Option<TypeIndex>,
&Transaction,
),
@@ -409,12 +407,12 @@ impl Indexer {
.p2aaddressindex_to_p2abytes
.get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
_ => {
unreachable!()
}
};
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
prev_addressbytes_opt.as_ref().ok_or(Error::Str("Expect to have addressbytes"))?;
if stores.addressbyteshash_to_typeindex.needs(height)
&& prev_addressbytes != addressbytes
@@ -452,7 +450,7 @@ impl Indexer {
))
},
)
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
.try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
@@ -476,20 +474,19 @@ impl Indexer {
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex =
txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle
.ok()
.context(
"Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join",
.map_err(|_|
Error::Str("Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join")
)??;
let input_source_vec = input_source_vec_handle
.ok()
.context("Export input_source_vec_handle to join")??;
.map_err(|_|
Error::Str("Export input_source_vec_handle to join")
)??;
let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt =
outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle
.ok()
.context(
"Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join",
.map_err(|_|
Error::Str("Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join")
)?;
let outputs_len = outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt.len();
@@ -510,7 +507,7 @@ impl Indexer {
outputindex,
(txout, txindex, vout, outputtype, addressbytes_res, typeindex_opt, _tx),
)|
-> color_eyre::Result<()> {
-> Result<()> {
let sats = Sats::from(txout.value);
if vout.is_zero() {
@@ -580,6 +577,7 @@ impl Indexer {
vecs.unknownoutputindex_to_txindex.push_if_needed(idxs.unknownoutputindex, txindex)?;
idxs.unknownoutputindex.copy_then_increment()
},
_ => unreachable!()
};
if let Ok(addressbytes) = addressbytes_res {
@@ -618,7 +616,7 @@ impl Indexer {
.into_iter()
.map(
#[allow(clippy::type_complexity)]
|(inputindex, input_source)| -> color_eyre::Result<(
|(inputindex, input_source)| -> Result<(
InputIndex, Vin, TxIndex, OutputIndex
)> {
match input_source {
@@ -634,7 +632,7 @@ impl Indexer {
let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.get(&TxidPrefix::from(&txid))
.context("txid should be in same block").inspect_err(|_| {
.ok_or(Error::Str("txid should be in same block")).inspect_err(|_| {
dbg!(&txid_prefix_to_txid_and_block_txindex_and_prev_txindex);
// panic!();
})?
@@ -643,7 +641,7 @@ impl Indexer {
let prev_outputindex = new_txindexvout_to_outputindex
.remove(&(prev_txindex, vout))
.context("should have found addressindex from same block")
.ok_or(Error::Str("should have found addressindex from same block"))
.inspect_err(|_| {
dbg!(&new_txindexvout_to_outputindex, txin, prev_txindex, vout, txid);
})?;
@@ -653,7 +651,7 @@ impl Indexer {
}
},
)
.try_for_each(|res| -> color_eyre::Result<()> {
.try_for_each(|res| -> Result<()> {
let (inputindex, vin, txindex, outputindex) = res?;
if vin.is_zero() {
@@ -675,7 +673,7 @@ impl Indexer {
txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.into_iter()
.try_for_each(
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> Result<()> {
let txindex = idxs.txindex + index;
txindex_to_tx_and_txid.insert(txindex, (tx, txid));
@@ -700,7 +698,7 @@ impl Indexer {
// Ok if `get` is not par as should happen only twice
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.context("To have txid for txindex")
.ok_or(Error::Str("To have txid for txindex"))
.inspect_err(|_| {
dbg!(txindex, len);
})?;
@@ -712,11 +710,13 @@ impl Indexer {
let only_known_dup_txids = [
bitcoin::Txid::from_str(
"d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599",
)?
)
.unwrap()
.into(),
bitcoin::Txid::from_str(
"e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468",
)?
)
.unwrap()
.into(),
];
@@ -724,7 +724,7 @@ impl Indexer {
if !is_dup {
dbg!(height, txindex, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
return Err(Error::Str("Expect none"));
}
}
}
@@ -737,13 +737,13 @@ impl Indexer {
txindex_to_tx_and_txid
.into_iter()
.try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
.try_for_each(|(txindex, (tx, txid))| -> Result<()> {
vecs.txindex_to_txversion.push_if_needed(txindex, tx.version.into())?;
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
vecs.txindex_to_rawlocktime.push_if_needed(txindex, tx.lock_time.into())?;
vecs.txindex_to_base_size.push_if_needed(txindex, tx.base_size().into())?;
vecs.txindex_to_total_size.push_if_needed(txindex, tx.total_size().into())?;
vecs.txindex_to_is_explicitly_rbf.push_if_needed(txindex, tx.is_explicitly_rbf())?;
vecs.txindex_to_is_explicitly_rbf.push_if_needed(txindex, StoredBool::from(tx.is_explicitly_rbf()))?;
Ok(())
})?;

15
crates/brk_indexer/src/stores.rs
View File

@@ -1,11 +1,12 @@
use std::{borrow::Cow, fs, path::Path, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex,
OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
};
use brk_error::Result;
use brk_store::{AnyStore, Store};
use brk_vecs::{AnyIterableVec, VecIterator};
use brk_structs::{
AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex,
OutputType, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
};
use brk_vecs::VecIterator;
use fjall::{PersistMode, TransactionalKeyspace};
use rayon::prelude::*;
@@ -27,7 +28,7 @@ pub struct Stores {
const VERSION: Version = Version::ZERO;
impl Stores {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
fs::create_dir_all(path)?;
let keyspace = match brk_store::open_keyspace(path) {
@@ -217,7 +218,7 @@ impl Stores {
&mut self,
vecs: &mut Vecs,
starting_indexes: &Indexes,
) -> color_eyre::Result<()> {
) -> Result<()> {
if self.addressbyteshash_to_typeindex.is_empty()?
&& self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?

347
crates/brk_indexer/src/vecs.rs
View File

@@ -1,14 +1,17 @@
use std::sync::Arc;
use brk_core::{
use brk_error::Result;
use brk_structs::{
AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, P2AAddressIndex, P2ABytes, P2MSOutputIndex, P2PK33AddressIndex, P2PK33Bytes,
P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex, P2PKHBytes, P2SHAddressIndex, P2SHBytes,
P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes,
RawLockTime, Result, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion,
RawLockTime, Sats, StoredBool, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion,
Txid, TypeIndex, UnknownOutputIndex, Version, Weight,
};
use brk_vecs::{AnyCollectableVec, AnyStampedVec, File, Format, StampedVec};
use brk_vecs::{
AnyCollectableVec, AnyStoredVec, CompressedVec, File, GenericStoredVec, RawVec, Stamp,
};
use rayon::prelude::*;
use crate::Indexes;
@@ -17,320 +20,276 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub emptyoutputindex_to_txindex: StampedVec<EmptyOutputIndex, TxIndex>,
pub height_to_blockhash: StampedVec<Height, BlockHash>,
pub height_to_difficulty: StampedVec<Height, StoredF64>,
pub height_to_first_emptyoutputindex: StampedVec<Height, EmptyOutputIndex>,
pub height_to_first_inputindex: StampedVec<Height, InputIndex>,
pub height_to_first_opreturnindex: StampedVec<Height, OpReturnIndex>,
pub height_to_first_outputindex: StampedVec<Height, OutputIndex>,
pub height_to_first_p2aaddressindex: StampedVec<Height, P2AAddressIndex>,
pub height_to_first_p2msoutputindex: StampedVec<Height, P2MSOutputIndex>,
pub height_to_first_p2pk33addressindex: StampedVec<Height, P2PK33AddressIndex>,
pub height_to_first_p2pk65addressindex: StampedVec<Height, P2PK65AddressIndex>,
pub height_to_first_p2pkhaddressindex: StampedVec<Height, P2PKHAddressIndex>,
pub height_to_first_p2shaddressindex: StampedVec<Height, P2SHAddressIndex>,
pub height_to_first_p2traddressindex: StampedVec<Height, P2TRAddressIndex>,
pub height_to_first_p2wpkhaddressindex: StampedVec<Height, P2WPKHAddressIndex>,
pub height_to_first_p2wshaddressindex: StampedVec<Height, P2WSHAddressIndex>,
pub height_to_first_txindex: StampedVec<Height, TxIndex>,
pub height_to_first_unknownoutputindex: StampedVec<Height, UnknownOutputIndex>,
pub emptyoutputindex_to_txindex: CompressedVec<EmptyOutputIndex, TxIndex>,
pub height_to_blockhash: RawVec<Height, BlockHash>,
pub height_to_difficulty: CompressedVec<Height, StoredF64>,
pub height_to_first_emptyoutputindex: CompressedVec<Height, EmptyOutputIndex>,
pub height_to_first_inputindex: CompressedVec<Height, InputIndex>,
pub height_to_first_opreturnindex: CompressedVec<Height, OpReturnIndex>,
pub height_to_first_outputindex: CompressedVec<Height, OutputIndex>,
pub height_to_first_p2aaddressindex: CompressedVec<Height, P2AAddressIndex>,
pub height_to_first_p2msoutputindex: CompressedVec<Height, P2MSOutputIndex>,
pub height_to_first_p2pk33addressindex: CompressedVec<Height, P2PK33AddressIndex>,
pub height_to_first_p2pk65addressindex: CompressedVec<Height, P2PK65AddressIndex>,
pub height_to_first_p2pkhaddressindex: CompressedVec<Height, P2PKHAddressIndex>,
pub height_to_first_p2shaddressindex: CompressedVec<Height, P2SHAddressIndex>,
pub height_to_first_p2traddressindex: CompressedVec<Height, P2TRAddressIndex>,
pub height_to_first_p2wpkhaddressindex: CompressedVec<Height, P2WPKHAddressIndex>,
pub height_to_first_p2wshaddressindex: CompressedVec<Height, P2WSHAddressIndex>,
pub height_to_first_txindex: CompressedVec<Height, TxIndex>,
pub height_to_first_unknownoutputindex: CompressedVec<Height, UnknownOutputIndex>,
/// Doesn't guarantee continuity due to possible reorgs
pub height_to_timestamp: StampedVec<Height, Timestamp>,
pub height_to_total_size: StampedVec<Height, StoredUsize>,
pub height_to_weight: StampedVec<Height, Weight>,
pub height_to_timestamp: CompressedVec<Height, Timestamp>,
pub height_to_total_size: CompressedVec<Height, StoredU64>,
pub height_to_weight: CompressedVec<Height, Weight>,
/// If outputindex == Outputindex::MAX then it's coinbase
pub inputindex_to_outputindex: StampedVec<InputIndex, OutputIndex>,
pub opreturnindex_to_txindex: StampedVec<OpReturnIndex, TxIndex>,
pub outputindex_to_outputtype: StampedVec<OutputIndex, OutputType>,
pub outputindex_to_typeindex: StampedVec<OutputIndex, TypeIndex>,
pub outputindex_to_value: StampedVec<OutputIndex, Sats>,
pub p2aaddressindex_to_p2abytes: StampedVec<P2AAddressIndex, P2ABytes>,
pub p2msoutputindex_to_txindex: StampedVec<P2MSOutputIndex, TxIndex>,
pub p2pk33addressindex_to_p2pk33bytes: StampedVec<P2PK33AddressIndex, P2PK33Bytes>,
pub p2pk65addressindex_to_p2pk65bytes: StampedVec<P2PK65AddressIndex, P2PK65Bytes>,
pub p2pkhaddressindex_to_p2pkhbytes: StampedVec<P2PKHAddressIndex, P2PKHBytes>,
pub p2shaddressindex_to_p2shbytes: StampedVec<P2SHAddressIndex, P2SHBytes>,
pub p2traddressindex_to_p2trbytes: StampedVec<P2TRAddressIndex, P2TRBytes>,
pub p2wpkhaddressindex_to_p2wpkhbytes: StampedVec<P2WPKHAddressIndex, P2WPKHBytes>,
pub p2wshaddressindex_to_p2wshbytes: StampedVec<P2WSHAddressIndex, P2WSHBytes>,
pub txindex_to_base_size: StampedVec<TxIndex, StoredU32>,
pub txindex_to_first_inputindex: StampedVec<TxIndex, InputIndex>,
pub txindex_to_first_outputindex: StampedVec<TxIndex, OutputIndex>,
pub txindex_to_is_explicitly_rbf: StampedVec<TxIndex, bool>,
pub txindex_to_rawlocktime: StampedVec<TxIndex, RawLockTime>,
pub txindex_to_total_size: StampedVec<TxIndex, StoredU32>,
pub txindex_to_txid: StampedVec<TxIndex, Txid>,
pub txindex_to_txversion: StampedVec<TxIndex, TxVersion>,
pub unknownoutputindex_to_txindex: StampedVec<UnknownOutputIndex, TxIndex>,
pub inputindex_to_outputindex: RawVec<InputIndex, OutputIndex>,
pub opreturnindex_to_txindex: CompressedVec<OpReturnIndex, TxIndex>,
pub outputindex_to_outputtype: RawVec<OutputIndex, OutputType>,
pub outputindex_to_typeindex: RawVec<OutputIndex, TypeIndex>,
pub outputindex_to_value: RawVec<OutputIndex, Sats>,
pub p2aaddressindex_to_p2abytes: RawVec<P2AAddressIndex, P2ABytes>,
pub p2msoutputindex_to_txindex: CompressedVec<P2MSOutputIndex, TxIndex>,
pub p2pk33addressindex_to_p2pk33bytes: RawVec<P2PK33AddressIndex, P2PK33Bytes>,
pub p2pk65addressindex_to_p2pk65bytes: RawVec<P2PK65AddressIndex, P2PK65Bytes>,
pub p2pkhaddressindex_to_p2pkhbytes: RawVec<P2PKHAddressIndex, P2PKHBytes>,
pub p2shaddressindex_to_p2shbytes: RawVec<P2SHAddressIndex, P2SHBytes>,
pub p2traddressindex_to_p2trbytes: RawVec<P2TRAddressIndex, P2TRBytes>,
pub p2wpkhaddressindex_to_p2wpkhbytes: RawVec<P2WPKHAddressIndex, P2WPKHBytes>,
pub p2wshaddressindex_to_p2wshbytes: RawVec<P2WSHAddressIndex, P2WSHBytes>,
pub txindex_to_base_size: CompressedVec<TxIndex, StoredU32>,
pub txindex_to_first_inputindex: CompressedVec<TxIndex, InputIndex>,
pub txindex_to_first_outputindex: CompressedVec<TxIndex, OutputIndex>,
pub txindex_to_is_explicitly_rbf: CompressedVec<TxIndex, StoredBool>,
pub txindex_to_rawlocktime: CompressedVec<TxIndex, RawLockTime>,
pub txindex_to_total_size: CompressedVec<TxIndex, StoredU32>,
pub txindex_to_txid: RawVec<TxIndex, Txid>,
pub txindex_to_txversion: CompressedVec<TxIndex, TxVersion>,
pub unknownoutputindex_to_txindex: CompressedVec<UnknownOutputIndex, TxIndex>,
}
impl Vecs {
pub fn forced_import(file: &Arc<File>, version: Version) -> color_eyre::Result<Self> {
pub fn forced_import(file: &Arc<File>, version: Version) -> Result<Self> {
Ok(Self {
emptyoutputindex_to_txindex: StampedVec::forced_import(
emptyoutputindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_blockhash: StampedVec::forced_import(
height_to_blockhash: RawVec::forced_import(
file,
"blockhash",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_difficulty: StampedVec::forced_import(
height_to_difficulty: CompressedVec::forced_import(
file,
"difficulty",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_emptyoutputindex: StampedVec::forced_import(
height_to_first_emptyoutputindex: CompressedVec::forced_import(
file,
"first_emptyoutputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_inputindex: StampedVec::forced_import(
height_to_first_inputindex: CompressedVec::forced_import(
file,
"first_inputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_opreturnindex: StampedVec::forced_import(
height_to_first_opreturnindex: CompressedVec::forced_import(
file,
"first_opreturnindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_outputindex: StampedVec::forced_import(
height_to_first_outputindex: CompressedVec::forced_import(
file,
"first_outputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2aaddressindex: StampedVec::forced_import(
height_to_first_p2aaddressindex: CompressedVec::forced_import(
file,
"first_p2aaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2msoutputindex: StampedVec::forced_import(
height_to_first_p2msoutputindex: CompressedVec::forced_import(
file,
"first_p2msoutputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2pk33addressindex: StampedVec::forced_import(
height_to_first_p2pk33addressindex: CompressedVec::forced_import(
file,
"first_p2pk33addressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2pk65addressindex: StampedVec::forced_import(
height_to_first_p2pk65addressindex: CompressedVec::forced_import(
file,
"first_p2pk65addressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2pkhaddressindex: StampedVec::forced_import(
height_to_first_p2pkhaddressindex: CompressedVec::forced_import(
file,
"first_p2pkhaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2shaddressindex: StampedVec::forced_import(
height_to_first_p2shaddressindex: CompressedVec::forced_import(
file,
"first_p2shaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2traddressindex: StampedVec::forced_import(
height_to_first_p2traddressindex: CompressedVec::forced_import(
file,
"first_p2traddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2wpkhaddressindex: StampedVec::forced_import(
height_to_first_p2wpkhaddressindex: CompressedVec::forced_import(
file,
"first_p2wpkhaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_p2wshaddressindex: StampedVec::forced_import(
height_to_first_p2wshaddressindex: CompressedVec::forced_import(
file,
"first_p2wshaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_txindex: StampedVec::forced_import(
height_to_first_txindex: CompressedVec::forced_import(
file,
"first_txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_first_unknownoutputindex: StampedVec::forced_import(
height_to_first_unknownoutputindex: CompressedVec::forced_import(
file,
"first_unknownoutputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_timestamp: StampedVec::forced_import(
height_to_timestamp: CompressedVec::forced_import(
file,
"timestamp",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_total_size: StampedVec::forced_import(
height_to_total_size: CompressedVec::forced_import(
file,
"total_size",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_weight: StampedVec::forced_import(
height_to_weight: CompressedVec::forced_import(
file,
"weight",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
inputindex_to_outputindex: StampedVec::forced_import(
inputindex_to_outputindex: RawVec::forced_import(
file,
"outputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
opreturnindex_to_txindex: StampedVec::forced_import(
opreturnindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
outputindex_to_outputtype: StampedVec::forced_import(
outputindex_to_outputtype: RawVec::forced_import(
file,
"outputtype",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
outputindex_to_typeindex: StampedVec::forced_import(
outputindex_to_typeindex: RawVec::forced_import(
file,
"typeindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
outputindex_to_value: StampedVec::forced_import(
outputindex_to_value: RawVec::forced_import(
file,
"value",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2aaddressindex_to_p2abytes: StampedVec::forced_import(
p2aaddressindex_to_p2abytes: RawVec::forced_import(
file,
"p2abytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2msoutputindex_to_txindex: StampedVec::forced_import(
p2msoutputindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk33addressindex_to_p2pk33bytes: StampedVec::forced_import(
p2pk33addressindex_to_p2pk33bytes: RawVec::forced_import(
file,
"p2pk33bytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk65addressindex_to_p2pk65bytes: StampedVec::forced_import(
p2pk65addressindex_to_p2pk65bytes: RawVec::forced_import(
file,
"p2pk65bytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pkhaddressindex_to_p2pkhbytes: StampedVec::forced_import(
p2pkhaddressindex_to_p2pkhbytes: RawVec::forced_import(
file,
"p2pkhbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2shaddressindex_to_p2shbytes: StampedVec::forced_import(
p2shaddressindex_to_p2shbytes: RawVec::forced_import(
file,
"p2shbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2traddressindex_to_p2trbytes: StampedVec::forced_import(
p2traddressindex_to_p2trbytes: RawVec::forced_import(
file,
"p2trbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wpkhaddressindex_to_p2wpkhbytes: StampedVec::forced_import(
p2wpkhaddressindex_to_p2wpkhbytes: RawVec::forced_import(
file,
"p2wpkhbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wshaddressindex_to_p2wshbytes: StampedVec::forced_import(
p2wshaddressindex_to_p2wshbytes: RawVec::forced_import(
file,
"p2wshbytes",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_base_size: StampedVec::forced_import(
txindex_to_base_size: CompressedVec::forced_import(
file,
"base_size",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_first_inputindex: StampedVec::forced_import(
txindex_to_first_inputindex: CompressedVec::forced_import(
file,
"first_inputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_first_outputindex: StampedVec::forced_import(
txindex_to_first_outputindex: CompressedVec::forced_import(
file,
"first_outputindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_is_explicitly_rbf: StampedVec::forced_import(
txindex_to_is_explicitly_rbf: CompressedVec::forced_import(
file,
"is_explicitly_rbf",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_rawlocktime: StampedVec::forced_import(
txindex_to_rawlocktime: CompressedVec::forced_import(
file,
"rawlocktime",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_total_size: StampedVec::forced_import(
txindex_to_total_size: CompressedVec::forced_import(
file,
"total_size",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_txid: StampedVec::forced_import(
txindex_to_txid: RawVec::forced_import(
file,
"txid",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_txversion: StampedVec::forced_import(
txindex_to_txversion: CompressedVec::forced_import(
file,
"txversion",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
unknownoutputindex_to_txindex: StampedVec::forced_import(
unknownoutputindex_to_txindex: CompressedVec::forced_import(
file,
"txindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
})
}
@@ -360,89 +319,93 @@ impl Vecs {
let stamp = u64::from(saved_height).into();
self.emptyoutputindex_to_txindex
.truncate_if_needed(emptyoutputindex, stamp)?;
self.height_to_blockhash.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(emptyoutputindex, stamp)?;
self.height_to_blockhash
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_difficulty
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_emptyoutputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_inputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_opreturnindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_outputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2aaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2msoutputindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2pk33addressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2pk65addressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2pkhaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2shaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2traddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2wpkhaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_p2wshaddressindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_txindex
.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_first_unknownoutputindex
.truncate_if_needed(height, stamp)?;
self.height_to_timestamp.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_timestamp
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_total_size
.truncate_if_needed(height, stamp)?;
self.height_to_weight.truncate_if_needed(height, stamp)?;
.truncate_if_needed_with_stamp(height, stamp)?;
self.height_to_weight
.truncate_if_needed_with_stamp(height, stamp)?;
self.inputindex_to_outputindex
.truncate_if_needed(inputindex, stamp)?;
.truncate_if_needed_with_stamp(inputindex, stamp)?;
self.opreturnindex_to_txindex
.truncate_if_needed(opreturnindex, stamp)?;
.truncate_if_needed_with_stamp(opreturnindex, stamp)?;
self.outputindex_to_outputtype
.truncate_if_needed(outputindex, stamp)?;
.truncate_if_needed_with_stamp(outputindex, stamp)?;
self.outputindex_to_typeindex
.truncate_if_needed(outputindex, stamp)?;
.truncate_if_needed_with_stamp(outputindex, stamp)?;
self.outputindex_to_value
.truncate_if_needed(outputindex, stamp)?;
.truncate_if_needed_with_stamp(outputindex, stamp)?;
self.p2aaddressindex_to_p2abytes
.truncate_if_needed(p2aaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2aaddressindex, stamp)?;
self.p2msoutputindex_to_txindex
.truncate_if_needed(p2msoutputindex, stamp)?;
.truncate_if_needed_with_stamp(p2msoutputindex, stamp)?;
self.p2pk33addressindex_to_p2pk33bytes
.truncate_if_needed(p2pk33addressindex, stamp)?;
.truncate_if_needed_with_stamp(p2pk33addressindex, stamp)?;
self.p2pk65addressindex_to_p2pk65bytes
.truncate_if_needed(p2pk65addressindex, stamp)?;
.truncate_if_needed_with_stamp(p2pk65addressindex, stamp)?;
self.p2pkhaddressindex_to_p2pkhbytes
.truncate_if_needed(p2pkhaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2pkhaddressindex, stamp)?;
self.p2shaddressindex_to_p2shbytes
.truncate_if_needed(p2shaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2shaddressindex, stamp)?;
self.p2traddressindex_to_p2trbytes
.truncate_if_needed(p2traddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2traddressindex, stamp)?;
self.p2wpkhaddressindex_to_p2wpkhbytes
.truncate_if_needed(p2wpkhaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2wpkhaddressindex, stamp)?;
self.p2wshaddressindex_to_p2wshbytes
.truncate_if_needed(p2wshaddressindex, stamp)?;
.truncate_if_needed_with_stamp(p2wshaddressindex, stamp)?;
self.txindex_to_base_size
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_first_inputindex
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_first_outputindex
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_is_explicitly_rbf
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_rawlocktime
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_total_size
.truncate_if_needed(txindex, stamp)?;
self.txindex_to_txid.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_txid
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.txindex_to_txversion
.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed_with_stamp(txindex, stamp)?;
self.unknownoutputindex_to_txindex
.truncate_if_needed(unknownoutputindex, stamp)?;
.truncate_if_needed_with_stamp(unknownoutputindex, stamp)?;
Ok(())
}
@@ -451,35 +414,37 @@ impl Vecs {
match bytes {
AddressBytes::P2PK65(bytes) => self
.p2pk65addressindex_to_p2pk65bytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2PK33(bytes) => self
.p2pk33addressindex_to_p2pk33bytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2PKH(bytes) => self
.p2pkhaddressindex_to_p2pkhbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2SH(bytes) => self
.p2shaddressindex_to_p2shbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2WPKH(bytes) => self
.p2wpkhaddressindex_to_p2wpkhbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2WSH(bytes) => self
.p2wshaddressindex_to_p2wshbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2TR(bytes) => self
.p2traddressindex_to_p2trbytes
.push_if_needed(index.into(), bytes),
.push_if_needed(index.into(), bytes)?,
AddressBytes::P2A(bytes) => self
.p2aaddressindex_to_p2abytes
.push_if_needed(index.into(), bytes),
}
.push_if_needed(index.into(), bytes)?,
};
Ok(())
}
pub fn flush(&mut self, height: Height) -> Result<()> {
self.mut_vecs()
.into_par_iter()
.try_for_each(|vec| vec.flush(u64::from(height).into()))
.try_for_each(|vec| vec.stamped_flush(Stamp::from(u64::from(height))))?;
Ok(())
}
pub fn starting_height(&mut self) -> Height {
@@ -542,7 +507,7 @@ impl Vecs {
]
}
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStampedVec> {
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStoredVec> {
vec![
&mut self.emptyoutputindex_to_txindex,
&mut self.height_to_blockhash,

4
crates/brk_interface/Cargo.toml
View File

@@ -8,11 +8,11 @@ homepage.workspace = true
repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_computer = { workspace = true }
brk_error = { workspace = true }
brk_indexer = { workspace = true }
brk_structs = { workspace = true }
brk_vecs = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
schemars = "1.0.4"
serde = { workspace = true }

Some files were not shown because too many files have changed in this diff Show More