diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..3fbd9f4d5 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,14 @@ +# Release profile: all targets get native CPU +[profile.release.target.'cfg()'] +rustflags = ["-C", "target-cpu=native"] + +# Release profile: x86_64 gets native CPU + extra features (overrides the above) +[profile.release.target.'cfg(target_arch = "x86_64")'] +rustflags = [ + "-C", "target-cpu=native", + "-C", "target-feature=+bmi1,+bmi2,+avx2" +] + +# Dist profile: x86_64 gets only extra features (no native CPU) +[profile.dist.target.'cfg(target_arch = "x86_64")'] +rustflags = ["-C", "target-feature=+bmi1,+bmi2,+avx2"] diff --git a/Cargo.lock b/Cargo.lock index d39e5d009..51de1dc30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,14 +18,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] -name = "aes" -version = "0.8.4" +name = "ahash" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "cipher", - "cpufeatures", + "getrandom 0.3.3", + "once_cell", + "version_check", + "zerocopy", ] [[package]] @@ -319,6 +321,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" +[[package]] +name = "better_io" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92fde17f91e7ba10b2a07f8dff29530b77144894bc6ae850fbc66e1276af0d28" + [[package]] name = "bincode" version = "2.0.1" @@ -476,8 +484,7 @@ dependencies = [ "brk_bundler", "brk_cli", "brk_computer", - "brk_core", - "brk_exit", + "brk_error", "brk_fetcher", "brk_indexer", "brk_interface", @@ -486,6 +493,7 @@ dependencies = [ "brk_parser", "brk_server", "brk_store", + "brk_structs", "brk_vecs", ] @@ -506,8 +514,6 @@ version = "0.0.83" dependencies = [ "bitcoincore-rpc", "brk_computer", - "brk_core", - "brk_exit", "brk_fetcher", "brk_indexer", "brk_logger", @@ -529,14 +535,13 @@ version = "0.0.83" dependencies = [ "bitcoin", "bitcoincore-rpc", - "brk_core", - "brk_exit", + "brk_error", "brk_fetcher", "brk_indexer", "brk_logger", "brk_parser", + "brk_structs", "brk_vecs", - "color-eyre", "derive_deref", "log", "rayon", @@ -546,42 +551,24 @@ dependencies = [ ] [[package]] -name = "brk_core" +name = "brk_error" version = "0.0.83" dependencies = [ - "bitcoin", "bitcoincore-rpc", - "byteview", - "derive_deref", + "brk_vecs", "fjall", "jiff", - "log", - "rapidhash", - "rlimit", - "serde", - "serde_bytes", - "serde_json", + "minreq", "zerocopy", - "zerocopy-derive", -] - -[[package]] -name = "brk_exit" -version = "0.0.83" -dependencies = [ - "brk_logger", - "ctrlc", - "log", - "parking_lot", ] [[package]] name = "brk_fetcher" version = "0.0.83" dependencies = [ - "brk_core", + "brk_error", "brk_logger", - "color-eyre", + "brk_structs", "log", "minreq", "serde_json", @@ -593,13 +580,12 @@ version = "0.0.83" dependencies = [ "bitcoin", "bitcoincore-rpc", - "brk_core", - "brk_exit", + "brk_error", "brk_logger", "brk_parser", "brk_store", + "brk_structs", "brk_vecs", - "color-eyre", "fjall", "log", "rayon", @@ -610,10 +596,10 @@ name = "brk_interface" version = "0.0.83" dependencies = [ "brk_computer", - "brk_core", + "brk_error", "brk_indexer", + "brk_structs", "brk_vecs", - "color-eyre", "derive_deref", "schemars 1.0.4", "serde", @@ -626,10 +612,10 @@ dependencies = [ name = "brk_logger" version = "0.0.83" dependencies = [ - "color-eyre", "env_logger", "jiff", "log", + "owo-colors", ] [[package]] @@ -648,7 +634,7 @@ version = "0.0.83" dependencies = [ "bitcoin", "bitcoincore-rpc", - "brk_core", + "brk_structs", "crossbeam", "derive_deref", "rayon", @@ -1019,8 +1005,7 @@ dependencies = [ "bitcoincore-rpc", "brk_bundler", "brk_computer", - "brk_core", - "brk_exit", + "brk_error", "brk_fetcher", "brk_indexer", "brk_interface", @@ -1030,10 +1015,11 @@ dependencies = [ "brk_vecs", "clap", "clap_derive", - "color-eyre", "jiff", "log", "minreq", + "owo-colors", + "quick_cache", "serde", "serde_json", "tokio", @@ -1046,7 +1032,8 @@ dependencies = [ name = "brk_store" version = "0.0.83" dependencies = [ - "brk_core", + "brk_error", + "brk_structs", "byteview", "fjall", "log", @@ -1064,24 +1051,50 @@ dependencies = [ "serde", ] +[[package]] +name = "brk_structs" +version = "0.0.83" +dependencies = [ + "bitcoin", + "bitcoincore-rpc", + "brk_error", + "brk_vecs", + "byteview", + "derive_deref", + "fjall", + "jiff", + "rapidhash", + "serde", + "serde_bytes", + "serde_json", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "brk_vecs" version = "0.0.83" dependencies = [ - "brk_core", - "brk_exit", - "clap", - "clap_derive", + "brk_vecs_macros", + "ctrlc", "libc", "log", "memmap2", "parking_lot", + "pco", "rayon", "serde", "serde_json", "zerocopy", "zerocopy-derive", - "zstd", +] + +[[package]] +name = "brk_vecs_macros" +version = "0.0.83" +dependencies = [ + "quote", + "syn 2.0.104", ] [[package]] @@ -1138,15 +1151,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" -[[package]] -name = "bzip2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" -dependencies = [ - "libbz2-rs-sys", -] - [[package]] name = "castaway" version = "0.2.4" @@ -1158,9 +1162,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.30" +version = "1.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2" dependencies = [ "jobserver", "libc", @@ -1205,30 +1209,20 @@ dependencies = [ "windows-link", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clap" -version = "4.5.41" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.41" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" dependencies = [ "anstream", "anstyle", @@ -1434,6 +1428,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.6" @@ -1547,12 +1547,6 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "deflate64" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" - [[package]] name = "deranged" version = "0.4.0" @@ -1614,7 +1608,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -1623,6 +1616,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" +[[package]] +name = "dtype_dispatch" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a5ccdfd6c5e7e2fea9c5cf256f2a08216047fab19c621c3da64e9ae4a1462d" + [[package]] name = "dunce" version = "1.0.5" @@ -1944,6 +1943,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1994,15 +2003,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - [[package]] name = "http" version = "1.3.1" @@ -2171,15 +2171,6 @@ dependencies = [ "libc", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "interval-heap" version = "0.0.5" @@ -2329,43 +2320,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "libbz2-rs-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb" - [[package]] name = "libc" version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" -[[package]] -name = "liblzma" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0791ab7e08ccc8e0ce893f6906eb2703ed8739d8e89b57c0714e71bad09024c8" -dependencies = [ - "liblzma-sys", -] - -[[package]] -name = "liblzma-sys" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "libredox" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360e552c93fa0e8152ab463bc4c4837fce76a225df11dfaeea66c313de5e61f7" +checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ "bitflags 2.9.1", "libc", @@ -2673,9 +2638,9 @@ dependencies = [ [[package]] name = "oxc-browserslist" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509ae58826553dbd720886b6a77e7bb2a953d28a52728a74cc1fe296d66fa9a2" +checksum = "e05d19022f54d3e0b8b1679c80f02d140e95e4308385eb247ba3168c299c81bb" dependencies = [ "bincode", "nom", @@ -2688,9 +2653,9 @@ dependencies = [ [[package]] name = "oxc-miette" -version = "2.3.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8f905f1293ce969f8b6f08f2f4e8a0fbfd26dec42a30ea98b436d2faf01f50f" +checksum = "31cfb121c9d3e0f9082856927f5cff9594279c91b544f4436e4bc971563caa60" dependencies = [ "cfg-if", "owo-colors", @@ -2702,9 +2667,9 @@ dependencies = [ [[package]] name = "oxc-miette-derive" -version = "2.3.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd092b10278cef8cc76dad34116988c2f0ca34bf9082de5513bacd95f4553d0" +checksum = "a6eabb57f935b454fbe0552ea0abaaf9eb0019b5fa05a7bbe7efd5bd8c765085" dependencies = [ "proc-macro2", "quote", @@ -3198,13 +3163,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] -name = "pbkdf2" -version = "0.12.2" +name = "pco" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "ab068b64f2c6f074cbdcafc80ebd83a27da92a3848deba2fabc21eba6691fc65" dependencies = [ - "digest", - "hmac", + "better_io", + "dtype_dispatch", + "half", + "rand_xoshiro", ] [[package]] @@ -3347,12 +3314,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "ppmd-rust" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -3399,8 +3360,10 @@ version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8565e62e02af316570d4b492f17af1481d6c07cea60f4e7edd71700da5052ba9" dependencies = [ + "ahash", "equivalent", "hashbrown 0.15.4", + "parking_lot", ] [[package]] @@ -3487,6 +3450,15 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rapidhash" version = "2.0.2" @@ -3610,15 +3582,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rlimit" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" -dependencies = [ - "libc", -] - [[package]] name = "rolldown-ariadne" version = "0.5.2" @@ -3852,9 +3815,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" dependencies = [ "indexmap 2.10.0", "itoa", @@ -4035,12 +3998,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "sugar_path" version = "1.2.0" @@ -4223,9 +4180,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.47.0" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", @@ -4276,9 +4233,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e06723639aaded957e5a80be250c1f82f274b9d23ebb4d94163668470623461c" +checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1" dependencies = [ "indexmap 2.10.0", "serde", @@ -5011,51 +4968,18 @@ dependencies = [ "syn 2.0.104", ] -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", -] - [[package]] name = "zip" version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aed4ac33e8eb078c89e6cbb1d5c4c7703ec6d299fc3e7c3695af8f8b423468b" dependencies = [ - "aes", "arbitrary", - "bzip2", - "constant_time_eq", "crc32fast", - "deflate64", "flate2", - "getrandom 0.3.3", - "hmac", "indexmap 2.10.0", - "liblzma", "memchr", - "pbkdf2", - "ppmd-rust", - "sha1", - "time", - "zeroize", "zopfli", - "zstd", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f2a0d330a..ea0196499 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,8 +28,8 @@ bitcoincore-rpc = "0.19.0" brk_bundler = { version = "0.0.83", path = "crates/brk_bundler" } brk_cli = { version = "0.0.83", path = "crates/brk_cli" } brk_computer = { version = "0.0.83", path = "crates/brk_computer" } -brk_core = { version = "0.0.83", path = "crates/brk_core" } -brk_exit = { version = "0.0.83", path = "crates/brk_exit" } +brk_structs = { version = "0.0.83", path = "crates/brk_structs" } +brk_error = { version = "0.0.83", path = "crates/brk_error" } brk_fetcher = { version = "0.0.83", path = "crates/brk_fetcher" } brk_indexer = { version = "0.0.83", path = "crates/brk_indexer" } brk_interface = { version = "0.0.83", path = "crates/brk_interface" } @@ -39,23 +39,24 @@ brk_parser = { version = "0.0.83", path = "crates/brk_parser" } brk_server = { version = "0.0.83", path = "crates/brk_server" } brk_store = { version = "0.0.83", path = "crates/brk_store" } brk_vecs = { version = "0.0.83", path = "crates/brk_vecs" } +brk_vecs_macros = { version = "0.0.83", path = "crates/brk_vecs_macros" } byteview = "=0.6.1" -clap = { version = "4.5.41", features = ["string"] } +clap = { version = "4.5.42", features = ["string"] } clap_derive = "4.5.41" -color-eyre = "0.6.5" derive_deref = "1.1.1" fjall = "2.11.2" jiff = "0.2.15" -log = { version = "0.4.27" } +log = "0.4.27" minreq = { version = "2.14.0", features = ["https", "serde_json"] } +owo-colors = "4.2.2" parking_lot = "0.12.4" rayon = "1.10.0" -serde = { version = "1.0.219" } +serde = "1.0.219" serde_bytes = "0.11.17" serde_derive = "1.0.219" -serde_json = { version = "1.0.141", features = ["float_roundtrip"] } -tokio = { version = "1.47.0", features = ["rt-multi-thread"] } -zerocopy = { version = "0.8.26" } +serde_json = { version = "1.0.142", features = ["float_roundtrip"] } +tokio = { version = "1.47.1", features = ["rt-multi-thread"] } +zerocopy = "0.8.26" zerocopy-derive = "0.8.26" [workspace.metadata.release] diff --git a/README.md b/README.md index d7a82eac5..47cc318c9 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network. @@ -63,8 +57,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi - [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates - [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a Bitcoin Research Kit instance - [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer -- [`brk_core`](https://crates.io/crates/brk_core): The Core (Structs and Errors) of the Bitcoin Research Kit -- [`brk_exit`](https://crates.io/crates/brk_exit): An exit blocker built on top of ctrlc +- [`brk_structs`](https://crates.io/crates/brk_structs): The Core (Structs and Errors) of the Bitcoin Research Kit - [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher - [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser - [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index 3c1f56510..b37651c85 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -11,47 +11,47 @@ version.workspace = true [features] full = [ "bundler", - "core", "computer", - "exit", + "error", "fetcher", "indexer", + "interface", "logger", "mcp", "parser", - "interface", "server", "store", + "structs", "vecs", ] bundler = ["brk_bundler"] -core = ["brk_core"] computer = ["brk_computer"] -exit = ["brk_exit"] +error = ["brk_error"] fetcher = ["brk_fetcher"] indexer = ["brk_indexer"] +interface = ["brk_interface"] logger = ["brk_logger"] mcp = ["brk_mcp"] parser = ["brk_parser"] -interface = ["brk_interface"] server = ["brk_server"] store = ["brk_store"] +structs = ["brk_structs"] vecs = ["brk_vecs"] [dependencies] brk_bundler = { workspace = true, optional = true } brk_cli = { workspace = true } -brk_core = { workspace = true, optional = true } brk_computer = { workspace = true, optional = true } -brk_exit = { workspace = true, optional = true } +brk_error = { workspace = true, optional = true } brk_fetcher = { workspace = true, optional = true } brk_indexer = { workspace = true, optional = true } +brk_interface = { workspace = true, optional = true } brk_logger = { workspace = true, optional = true } brk_mcp = { workspace = true, optional = true } brk_parser = { workspace = true, optional = true } -brk_interface = { workspace = true, optional = true } brk_server = { workspace = true, optional = true } brk_store = { workspace = true, optional = true } +brk_structs = { workspace = true, optional = true } brk_vecs = { workspace = true, optional = true } [package.metadata.docs.rs] diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs index 1b7f87a0a..b117391f8 100644 --- a/crates/brk/src/lib.rs +++ b/crates/brk/src/lib.rs @@ -7,17 +7,17 @@ pub use brk_bundler as bundler; #[doc(inline)] pub use brk_cli as cli; -#[cfg(feature = "core")] +#[cfg(feature = "structs")] #[doc(inline)] -pub use brk_core as core; +pub use brk_structs as structs; #[cfg(feature = "computer")] #[doc(inline)] pub use brk_computer as computer; -#[cfg(feature = "exit")] +#[cfg(feature = "error")] #[doc(inline)] -pub use brk_exit as exit; +pub use brk_error as error; #[cfg(feature = "fetcher")] #[doc(inline)] diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 3f7a68ec1..da73ca3bb 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -10,8 +10,6 @@ repository.workspace = true [dependencies] bitcoincore-rpc = { workspace = true } brk_computer = { workspace = true } -brk_core = { workspace = true } -brk_exit = { workspace = true } brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } @@ -20,11 +18,11 @@ brk_server = { workspace = true } brk_vecs = { workspace = true } clap = { workspace = true } clap_derive = { workspace = true } -color-eyre = { workspace = true } +color-eyre = "0.6.5" log = { workspace = true } serde = { workspace = true } tokio = { workspace = true } -toml = "0.9.3" +toml = "0.9.4" [[bin]] name = "brk" diff --git a/crates/brk_cli/README.md b/crates/brk_cli/README.md index 22752a617..023057bb4 100644 --- a/crates/brk_cli/README.md +++ b/crates/brk_cli/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A command line interface to run a Bitcoin Research Kit instance. diff --git a/crates/brk_cli/src/config.rs b/crates/brk_cli/src/config.rs index a89d3ac53..e74571963 100644 --- a/crates/brk_cli/src/config.rs +++ b/crates/brk_cli/src/config.rs @@ -4,14 +4,16 @@ use std::{ }; use bitcoincore_rpc::{self, Auth, Client}; -use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path}; use brk_fetcher::Fetcher; use brk_server::Website; -use brk_vecs::{Computation, Format}; use clap::Parser; use clap_derive::Parser; use color_eyre::eyre::eyre; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::{default_bitcoin_path, default_brk_path, dot_brk_path}; + +const DOWNLOADS: &str = "downloads"; #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] #[command(version, about)] @@ -31,16 +33,6 @@ pub struct Config { #[arg(long, value_name = "PATH")] brkdir: Option, - /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved - #[serde(default, deserialize_with = "default_on_error")] - #[arg(short, long)] - computation: Option, - - /// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved - #[serde(default, deserialize_with = "default_on_error")] - #[arg(short, long)] - format: Option, - /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved #[serde(default, deserialize_with = "default_on_error")] #[arg(short = 'F', long, value_name = "BOOL")] @@ -122,18 +114,10 @@ impl Config { config_saved.brkdir = Some(brkdir); } - if let Some(computation) = config_args.computation.take() { - config_saved.computation = Some(computation); - } - if let Some(fetch) = config_args.fetch.take() { config_saved.fetch = Some(fetch); } - if let Some(format) = config_args.format.take() { - config_saved.format = Some(format); - } - if let Some(website) = config_args.website.take() { config_saved.website = Some(website); } @@ -291,6 +275,10 @@ impl Config { self.brkdir().join("hars") } + pub fn downloads_dir(&self) -> PathBuf { + dot_brk_path().join(DOWNLOADS) + } + fn path_cookiefile(&self) -> PathBuf { self.rpccookiefile.as_ref().map_or_else( || self.bitcoindir().join(".cookie"), @@ -329,14 +317,6 @@ impl Config { .then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap()) } - pub fn computation(&self) -> Computation { - self.computation.unwrap_or_default() - } - - pub fn format(&self) -> Format { - self.format.unwrap_or_default() - } - pub fn check_collisions(&self) -> bool { self.check_collisions.is_some_and(|b| b) } @@ -349,3 +329,14 @@ impl Config { self.watch.is_some_and(|b| b) } } + +fn default_on_error<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: Deserialize<'de> + Default, +{ + match T::deserialize(deserializer) { + Ok(v) => Ok(v), + Err(_) => Ok(T::default()), + } +} diff --git a/crates/brk_cli/src/lib.rs b/crates/brk_cli/src/lib.rs index 2e143465d..b96bc45af 100644 --- a/crates/brk_cli/src/lib.rs +++ b/crates/brk_cli/src/lib.rs @@ -2,11 +2,11 @@ use std::{fs, thread}; -use brk_core::{dot_brk_log_path, dot_brk_path}; - mod config; +mod paths; mod run; +pub use paths::*; use run::*; pub fn main() -> color_eyre::Result<()> { diff --git a/crates/brk_core/src/utils/paths.rs b/crates/brk_cli/src/paths.rs similarity index 100% rename from crates/brk_core/src/utils/paths.rs rename to crates/brk_cli/src/paths.rs diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index 41328e299..aa5a0c22b 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -2,9 +2,9 @@ use std::{thread::sleep, time::Duration}; use bitcoincore_rpc::{self, RpcApi}; use brk_computer::Computer; -use brk_exit::Exit; use brk_indexer::Indexer; use brk_server::Server; +use brk_vecs::Exit; use log::info; use crate::config::Config; @@ -13,10 +13,11 @@ pub fn run() -> color_eyre::Result<()> { let config = Config::import()?; let rpc = config.rpc()?; - let exit = Exit::new(); - let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc); - let format = config.format(); + let exit = Exit::new(); + exit.set_ctrlc_handler(); + + let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc); let mut indexer = Indexer::forced_import(&config.brkdir())?; @@ -36,13 +37,7 @@ pub fn run() -> color_eyre::Result<()> { Ok(()) }; - let mut computer = Computer::forced_import( - &config.brkdir(), - &indexer, - config.computation(), - config.fetcher(), - format, - )?; + let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?; tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -51,7 +46,12 @@ pub fn run() -> color_eyre::Result<()> { let served_indexer = indexer.clone(); let served_computer = computer.clone(); - let server = Server::new(served_indexer, served_computer, config.website())?; + let server = Server::new( + served_indexer, + served_computer, + config.website(), + &config.downloads_dir(), + )?; let watch = config.watch(); let mcp = config.mcp(); @@ -72,7 +72,7 @@ pub fn run() -> color_eyre::Result<()> { let starting_indexes = indexer.index(&parser, rpc, &exit, config.check_collisions())?; - computer.compute(&mut indexer, starting_indexes, &exit)?; + computer.compute(&indexer, starting_indexes, &exit)?; if let Some(delay) = config.delay() { sleep(Duration::from_secs(delay)) diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 0eabffa43..355060750 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -10,14 +10,13 @@ repository.workspace = true [dependencies] bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } -brk_core = { workspace = true } -brk_exit = { workspace = true } +brk_structs = { workspace = true } +brk_error = { workspace = true } brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_vecs = { workspace = true } -color-eyre = { workspace = true } derive_deref = { workspace = true } log = { workspace = true } rayon = { workspace = true } diff --git a/crates/brk_computer/README.md b/crates/brk_computer/README.md index 55d7ad3ee..bba172db6 100644 --- a/crates/brk_computer/README.md +++ b/crates/brk_computer/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A dataset computer, built on top of `brk_indexer` and `brk_fetcher`. It computes any dataset you can think of and if it doesn't feel free to create an issue. diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index f0140d70a..3dc8b3d32 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -1,18 +1,16 @@ use std::{path::Path, thread}; use brk_computer::Computer; -use brk_exit::Exit; +use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::Parser; -use brk_vecs::{Computation, Format}; - -pub fn main() -> color_eyre::Result<()> { - color_eyre::install()?; +use brk_vecs::Exit; +pub fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); - // let bitcoin_dir = brk_core::default_bitcoin_path(); + // let bitcoin_dir = brk_structs::default_bitcoin_path(); let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin"); let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( @@ -20,38 +18,27 @@ pub fn main() -> color_eyre::Result<()> { bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), )?)); let exit = Exit::new(); + exit.set_ctrlc_handler(); // Can't increase main thread's stack programatically, thus we need to use another thread thread::Builder::new() .stack_size(256 * 1024 * 1024) - .spawn(move || -> color_eyre::Result<()> { - let parser = Parser::new( - bitcoin_dir.join("blocks"), - brk_core::default_brk_path(), - rpc, - ); + .spawn(move || -> Result<()> { + let parser = Parser::new(bitcoin_dir.join("blocks"), Path::new("").to_path_buf(), rpc); let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs"); let outputs_dir = _outputs_dir.as_path(); // let outputs_dir = Path::new("../../_outputs"); - let format = Format::Raw; - let mut indexer = Indexer::forced_import(outputs_dir)?; let fetcher = Fetcher::import(None)?; - let mut computer = Computer::forced_import( - outputs_dir, - &indexer, - Computation::Lazy, - Some(fetcher), - format, - )?; + let mut computer = Computer::forced_import(outputs_dir, &indexer, Some(fetcher))?; let starting_indexes = indexer.index(&parser, rpc, &exit, true)?; - computer.compute(&mut indexer, starting_indexes, &exit)?; + computer.compute(&indexer, starting_indexes, &exit)?; Ok(()) })? diff --git a/crates/brk_computer/src/all.rs b/crates/brk_computer/src/all.rs index d6e003cd7..1a08b7745 100644 --- a/crates/brk_computer/src/all.rs +++ b/crates/brk_computer/src/all.rs @@ -1,13 +1,13 @@ use std::{path::Path, sync::Arc}; -use brk_core::Version; -use brk_exit::Exit; +use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, Computation, File, Format}; +use brk_structs::Version; +use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format}; use log::info; -use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions}; +use crate::{blocks, cointime, constants, fetched, indexes, market, mining, price, transactions}; use super::stateful; @@ -20,6 +20,7 @@ pub struct Vecs { pub blocks: blocks::Vecs, pub mining: mining::Vecs, pub market: market::Vecs, + pub price: Option, pub transactions: transactions::Vecs, pub stateful: stateful::Vecs, pub fetched: Option, @@ -32,12 +33,12 @@ impl Vecs { file: &Arc, version: Version, indexer: &Indexer, - fetch: bool, + fetcher: Option, computation: Computation, format: Format, fetched_file: &Arc, states_path: &Path, - ) -> color_eyre::Result { + ) -> Result { let indexes = indexes::Vecs::forced_import( file, version + VERSION + Version::ZERO, @@ -46,10 +47,19 @@ impl Vecs { format, )?; - let fetched = fetch.then(|| { + let fetched = fetcher.map(|fetcher| { fetched::Vecs::forced_import( file, fetched_file, + fetcher, + version + VERSION + Version::ZERO, + ) + .unwrap() + }); + + let price = fetched.is_some().then(|| { + price::Vecs::forced_import( + file, version + VERSION + Version::ZERO, computation, format, @@ -93,7 +103,7 @@ impl Vecs { computation, format, &indexes, - fetched.as_ref(), + price.as_ref(), states_path, )?, transactions: transactions::Vecs::forced_import( @@ -103,7 +113,7 @@ impl Vecs { &indexes, computation, format, - fetched.as_ref(), + price.as_ref(), )?, cointime: cointime::Vecs::forced_import( file, @@ -111,10 +121,11 @@ impl Vecs { computation, format, &indexes, - fetched.as_ref(), + price.as_ref(), )?, indexes, fetched, + price, }) } @@ -122,9 +133,8 @@ impl Vecs { &mut self, indexer: &Indexer, starting_indexes: brk_indexer::Indexes, - fetcher: Option<&mut Fetcher>, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { info!("Computing indexes..."); let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; @@ -142,11 +152,13 @@ impl Vecs { if let Some(fetched) = self.fetched.as_mut() { info!("Computing fetched..."); - fetched.compute( + fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?; + + self.price.as_mut().unwrap().compute( indexer, &self.indexes, &starting_indexes, - fetcher.unwrap(), + fetched, exit, )?; } @@ -156,16 +168,16 @@ impl Vecs { indexer, &self.indexes, &starting_indexes, - self.fetched.as_ref(), + self.price.as_ref(), exit, )?; - if let Some(fetched) = self.fetched.as_ref() { + if let Some(price) = self.price.as_ref() { info!("Computing market..."); self.market.compute( indexer, &self.indexes, - fetched, + price, &mut self.transactions, &starting_indexes, exit, @@ -177,7 +189,7 @@ impl Vecs { indexer, &self.indexes, &self.transactions, - self.fetched.as_ref(), + self.price.as_ref(), &self.market, &mut starting_indexes, exit, @@ -187,7 +199,7 @@ impl Vecs { indexer, &self.indexes, &starting_indexes, - self.fetched.as_ref(), + self.price.as_ref(), &self.transactions, &self.stateful, exit, @@ -207,6 +219,7 @@ impl Vecs { self.stateful.vecs(), self.cointime.vecs(), self.fetched.as_ref().map_or(vec![], |v| v.vecs()), + self.price.as_ref().map_or(vec![], |v| v.vecs()), ] .into_iter() .flatten() diff --git a/crates/brk_computer/src/blocks.rs b/crates/brk_computer/src/blocks.rs index f2c8b1074..ad8cc49fd 100644 --- a/crates/brk_computer/src/blocks.rs +++ b/crates/brk_computer/src/blocks.rs @@ -1,12 +1,12 @@ -use std::sync::Arc; +use std::{path::Path, sync::Arc}; -use brk_core::{ - CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize, - Timestamp, Version, Weight, -}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, File, Format}; +use brk_structs::{ + CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version, + Weight, +}; +use brk_vecs::{AnyCollectableVec, Computation, EagerVec, Exit, File, Format, VecIterator}; use crate::grouped::Source; @@ -20,6 +20,8 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { + file: Arc, + pub height_to_interval: EagerVec, pub height_to_vbytes: EagerVec, pub difficultyepoch_to_timestamp: EagerVec, @@ -27,28 +29,30 @@ pub struct Vecs { pub timeindexes_to_timestamp: ComputedVecsFromDateIndex, pub indexes_to_block_count: ComputedVecsFromHeight, pub indexes_to_block_interval: ComputedVecsFromHeight, - pub indexes_to_block_size: ComputedVecsFromHeight, + pub indexes_to_block_size: ComputedVecsFromHeight, pub indexes_to_block_vbytes: ComputedVecsFromHeight, pub indexes_to_block_weight: ComputedVecsFromHeight, } impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, - ) -> color_eyre::Result { + ) -> Result { + let file = Arc::new(File::open(&parent.join("blocks"))?); + Ok(Self { height_to_interval: EagerVec::forced_import( - file, + &file, "interval", version + VERSION + Version::ZERO, format, )?, timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import( - file, + &file, "timestamp", Source::Compute, version + VERSION + Version::ZERO, @@ -58,7 +62,7 @@ impl Vecs { VecBuilderOptions::default().add_first(), )?, indexes_to_block_interval: ComputedVecsFromHeight::forced_import( - file, + &file, "block_interval", Source::None, version + VERSION + Version::ZERO, @@ -71,7 +75,7 @@ impl Vecs { .add_average(), )?, indexes_to_block_count: ComputedVecsFromHeight::forced_import( - file, + &file, "block_count", Source::Compute, version + VERSION + Version::ZERO, @@ -81,7 +85,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_weight: ComputedVecsFromHeight::forced_import( - file, + &file, "block_weight", Source::None, version + VERSION + Version::ZERO, @@ -91,7 +95,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_size: ComputedVecsFromHeight::forced_import( - file, + &file, "block_size", Source::None, version + VERSION + Version::ZERO, @@ -101,13 +105,13 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, height_to_vbytes: EagerVec::forced_import( - file, + &file, "vbytes", version + VERSION + Version::ZERO, format, )?, indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( - file, + &file, "block_vbytes", Source::None, version + VERSION + Version::ZERO, @@ -117,17 +121,19 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, difficultyepoch_to_timestamp: EagerVec::forced_import( - file, + &file, "timestamp", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_timestamp: EagerVec::forced_import( - file, + &file, "timestamp", version + VERSION + Version::ZERO, format, )?, + + file, }) } @@ -137,7 +143,7 @@ impl Vecs { indexes: &indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.timeindexes_to_timestamp.compute_all( indexer, indexes, @@ -149,7 +155,8 @@ impl Vecs { &indexes.dateindex_to_date, |(di, d, ..)| (di, Timestamp::from(d)), exit, - ) + )?; + Ok(()) }, )?; @@ -164,7 +171,8 @@ impl Vecs { &indexer.vecs.height_to_weight, |h| (h, StoredU32::from(1_u32)), exit, - ) + )?; + Ok(()) }, )?; @@ -240,6 +248,8 @@ impl Vecs { exit, )?; + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index bf8326322..f244b6266 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -1,23 +1,25 @@ -use std::sync::Arc; +use std::{path::Path, sync::Arc}; -use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator}; +use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version}; +use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator}; use super::{ - Indexes, fetched, + Indexes, grouped::{ ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight, Source, VecBuilderOptions, }, - indexes, stateful, transactions, + indexes, price, stateful, transactions, }; const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { + file: Arc, + pub indexes_to_coinblocks_created: ComputedVecsFromHeight, pub indexes_to_coinblocks_stored: ComputedVecsFromHeight, pub indexes_to_liveliness: ComputedVecsFromHeight, @@ -46,18 +48,20 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); + price: Option<&price::Vecs>, + ) -> Result { + let file = Arc::new(File::open(&parent.join("cointime"))?); + + let compute_dollars = price.is_some(); Ok(Self { indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import( - file, + &file, "coinblocks_created", Source::Compute, version + VERSION + Version::ZERO, @@ -67,7 +71,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import( - file, + &file, "coinblocks_stored", Source::Compute, version + VERSION + Version::ZERO, @@ -77,7 +81,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_liveliness: ComputedVecsFromHeight::forced_import( - file, + &file, "liveliness", Source::Compute, version + VERSION + Version::ZERO, @@ -87,7 +91,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaultedness: ComputedVecsFromHeight::forced_import( - file, + &file, "vaultedness", Source::Compute, version + VERSION + Version::ZERO, @@ -97,7 +101,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import( - file, + &file, "activity_to_vaultedness_ratio", Source::Compute, version + VERSION + Version::ZERO, @@ -107,7 +111,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import( - file, + &file, "vaulted_supply", Source::Compute, version + VERSION + Version::ONE, @@ -118,7 +122,7 @@ impl Vecs { indexes, )?, indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import( - file, + &file, "active_supply", Source::Compute, version + VERSION + Version::ONE, @@ -129,7 +133,7 @@ impl Vecs { indexes, )?, indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import( - file, + &file, "thermo_cap", Source::Compute, version + VERSION + Version::ONE, @@ -139,7 +143,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_investor_cap: ComputedVecsFromHeight::forced_import( - file, + &file, "investor_cap", Source::Compute, version + VERSION + Version::ONE, @@ -149,7 +153,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import( - file, + &file, "vaulted_cap", Source::Compute, version + VERSION + Version::ONE, @@ -159,7 +163,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_active_cap: ComputedVecsFromHeight::forced_import( - file, + &file, "active_cap", Source::Compute, version + VERSION + Version::ONE, @@ -169,7 +173,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import( - file, + &file, "vaulted_price", Source::Compute, version + VERSION + Version::ZERO, @@ -179,7 +183,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "vaulted_price", Source::None, version + VERSION + Version::ZERO, @@ -189,7 +193,7 @@ impl Vecs { true, )?, indexes_to_active_price: ComputedVecsFromHeight::forced_import( - file, + &file, "active_price", Source::Compute, version + VERSION + Version::ZERO, @@ -199,7 +203,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "active_price", Source::None, version + VERSION + Version::ZERO, @@ -209,7 +213,7 @@ impl Vecs { true, )?, indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import( - file, + &file, "true_market_mean", Source::Compute, version + VERSION + Version::ZERO, @@ -219,7 +223,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "true_market_mean", Source::None, version + VERSION + Version::ZERO, @@ -229,7 +233,7 @@ impl Vecs { true, )?, indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import( - file, + &file, "cointime_value_destroyed", Source::Compute, version + VERSION + Version::ZERO, @@ -239,7 +243,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import( - file, + &file, "cointime_value_created", Source::Compute, version + VERSION + Version::ZERO, @@ -249,7 +253,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import( - file, + &file, "cointime_value_stored", Source::Compute, version + VERSION + Version::ZERO, @@ -259,7 +263,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_price: ComputedVecsFromHeight::forced_import( - file, + &file, "cointime_price", Source::Compute, version + VERSION + Version::ZERO, @@ -269,7 +273,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import( - file, + &file, "cointime_cap", Source::Compute, version + VERSION + Version::ZERO, @@ -279,7 +283,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "cointime_price", Source::None, version + VERSION + Version::ZERO, @@ -288,6 +292,8 @@ impl Vecs { indexes, true, )?, + + file, }) } @@ -297,11 +303,11 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, starting_indexes: &Indexes, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, transactions: &transactions::Vecs, stateful: &stateful::Vecs, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply; self.indexes_to_coinblocks_created.compute_all( @@ -315,7 +321,8 @@ impl Vecs { circulating_supply, |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), exit, - ) + )?; + Ok(()) }, )?; @@ -341,7 +348,8 @@ impl Vecs { (i, created.checked_sub(destroyed).unwrap()) }, exit, - ) + )?; + Ok(()) }, )?; @@ -360,7 +368,8 @@ impl Vecs { .height_extra .unwrap_cumulative(), exit, - ) + )?; + Ok(()) }, )?; let liveliness = &self.indexes_to_liveliness; @@ -376,7 +385,8 @@ impl Vecs { liveliness.height.as_ref().unwrap(), |(i, v, ..)| (i, StoredF64::from(1.0).checked_sub(v).unwrap()), exit, - ) + )?; + Ok(()) }, )?; let vaultedness = &self.indexes_to_vaultedness; @@ -392,14 +402,15 @@ impl Vecs { liveliness.height.as_ref().unwrap(), vaultedness.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_vaulted_supply.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { @@ -408,14 +419,15 @@ impl Vecs { circulating_supply, vaultedness.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_active_supply.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { @@ -424,11 +436,12 @@ impl Vecs { circulating_supply, liveliness.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; - if let Some(fetched) = fetched { + if let Some(price) = price { let realized_cap = stateful .utxo_cohorts .all @@ -465,7 +478,8 @@ impl Vecs { .unwrap_cumulative(), |(i, v, ..)| (i, v), exit, - ) + )?; + Ok(()) }, )?; @@ -480,7 +494,8 @@ impl Vecs { realized_cap, self.indexes_to_thermo_cap.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -495,7 +510,8 @@ impl Vecs { realized_cap, self.indexes_to_vaultedness.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -510,7 +526,8 @@ impl Vecs { realized_cap, self.indexes_to_liveliness.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -525,14 +542,15 @@ impl Vecs { realized_price, self.indexes_to_vaultedness.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_vaulted_price_ratio.compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(self.indexes_to_vaulted_price.dateindex.unwrap_last()), @@ -549,14 +567,15 @@ impl Vecs { realized_price, self.indexes_to_liveliness.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_active_price_ratio.compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(self.indexes_to_active_price.dateindex.unwrap_last()), @@ -577,14 +596,15 @@ impl Vecs { .as_ref() .unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_true_market_mean_ratio.compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(self.indexes_to_true_market_mean.dateindex.unwrap_last()), @@ -600,10 +620,11 @@ impl Vecs { // The price taken won't be correct for time based indexes vec.compute_multiply( starting_indexes.height, - &fetched.chainindexes_to_close.height, + &price.chainindexes_to_close.height, indexes_to_coinblocks_destroyed.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -615,10 +636,11 @@ impl Vecs { |vec, _, _, starting_indexes, exit| { vec.compute_multiply( starting_indexes.height, - &fetched.chainindexes_to_close.height, + &price.chainindexes_to_close.height, self.indexes_to_coinblocks_created.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -630,10 +652,11 @@ impl Vecs { |vec, _, _, starting_indexes, exit| { vec.compute_multiply( starting_indexes.height, - &fetched.chainindexes_to_close.height, + &price.chainindexes_to_close.height, self.indexes_to_coinblocks_stored.height.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -652,7 +675,8 @@ impl Vecs { .height_extra .unwrap_cumulative(), exit, - ) + )?; + Ok(()) }, )?; @@ -667,20 +691,23 @@ impl Vecs { self.indexes_to_cointime_price.height.as_ref().unwrap(), circulating_supply, exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_cointime_price_ratio.compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(self.indexes_to_cointime_price.dateindex.unwrap_last()), )?; } + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index 769e4da8f..5dc15b46f 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -1,9 +1,9 @@ -use std::sync::Arc; +use std::{path::Path, sync::Arc}; -use brk_core::{StoredU8, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, AnyVec, Computation, File, Format}; +use brk_structs::{StoredU16, Version}; +use brk_vecs::{AnyCollectableVec, AnyVec, Computation, Exit, File, Format}; use crate::grouped::Source; @@ -17,23 +17,27 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - pub constant_0: ComputedVecsFromHeight, - pub constant_1: ComputedVecsFromHeight, - pub constant_50: ComputedVecsFromHeight, - pub constant_100: ComputedVecsFromHeight, + file: Arc, + + pub constant_0: ComputedVecsFromHeight, + pub constant_1: ComputedVecsFromHeight, + pub constant_50: ComputedVecsFromHeight, + pub constant_100: ComputedVecsFromHeight, } impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, - ) -> color_eyre::Result { + ) -> Result { + let file = Arc::new(File::open(&parent.join("constants"))?); + Ok(Self { constant_0: ComputedVecsFromHeight::forced_import( - file, + &file, "constant_0", Source::Compute, version + VERSION + Version::ZERO, @@ -43,7 +47,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_1: ComputedVecsFromHeight::forced_import( - file, + &file, "constant_1", Source::Compute, version + VERSION + Version::ZERO, @@ -53,7 +57,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_50: ComputedVecsFromHeight::forced_import( - file, + &file, "constant_50", Source::Compute, version + VERSION + Version::ZERO, @@ -63,7 +67,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_100: ComputedVecsFromHeight::forced_import( - file, + &file, "constant_100", Source::Compute, version + VERSION + Version::ZERO, @@ -72,6 +76,8 @@ impl Vecs { indexes, VecBuilderOptions::default().add_last(), )?, + + file, }) } @@ -81,7 +87,7 @@ impl Vecs { indexes: &indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.constant_0.compute_all( indexer, indexes, @@ -92,9 +98,10 @@ impl Vecs { starting_indexes.height, indexes.height_to_date.len(), indexes.height_to_date.version(), - |i| (i, StoredU8::new(0)), + |i| (i, StoredU16::new(0)), exit, - ) + )?; + Ok(()) }, )?; @@ -108,9 +115,10 @@ impl Vecs { starting_indexes.height, indexes.height_to_date.len(), indexes.height_to_date.version(), - |i| (i, StoredU8::new(1)), + |i| (i, StoredU16::new(1)), exit, - ) + )?; + Ok(()) }, )?; @@ -124,9 +132,10 @@ impl Vecs { starting_indexes.height, indexes.height_to_date.len(), indexes.height_to_date.version(), - |i| (i, StoredU8::new(50)), + |i| (i, StoredU16::new(50)), exit, - ) + )?; + Ok(()) }, )?; @@ -140,12 +149,15 @@ impl Vecs { starting_indexes.height, indexes.height_to_date.len(), indexes.height_to_date.version(), - |i| (i, StoredU8::new(100)), + |i| (i, StoredU16::new(100)), exit, - ) + )?; + Ok(()) }, )?; + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index 068349c97..e1ac3a471 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -1,395 +1,44 @@ -use std::sync::Arc; +use std::{path::Path, sync::Arc}; -use brk_core::{ - Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex, - OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, - YearIndex, -}; -use brk_exit::Exit; +use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; +use brk_structs::{DateIndex, Height, OHLCCents, Version}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, StoredIndex, - VecIterator, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Exit, File, GenericStoredVec, RawVec, + StoredIndex, VecIterator, }; -use crate::grouped::Source; - -use super::{ - Indexes, - grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeightStrict, VecBuilderOptions}, - indexes, -}; +use super::{Indexes, indexes}; #[derive(Clone)] pub struct Vecs { - pub dateindex_to_close_in_cents: EagerVec>, - pub dateindex_to_high_in_cents: EagerVec>, - pub dateindex_to_low_in_cents: EagerVec>, - pub dateindex_to_ohlc: EagerVec, - pub dateindex_to_ohlc_in_sats: EagerVec, - pub dateindex_to_ohlc_in_cents: EagerVec, - pub dateindex_to_open_in_cents: EagerVec>, - pub height_to_close_in_cents: EagerVec>, - pub height_to_high_in_cents: EagerVec>, - pub height_to_low_in_cents: EagerVec>, - pub height_to_ohlc: EagerVec, - pub height_to_ohlc_in_sats: EagerVec, - pub height_to_ohlc_in_cents: EagerVec, - pub height_to_open_in_cents: EagerVec>, - pub timeindexes_to_close: ComputedVecsFromDateIndex>, - pub timeindexes_to_high: ComputedVecsFromDateIndex>, - pub timeindexes_to_low: ComputedVecsFromDateIndex>, - pub timeindexes_to_open: ComputedVecsFromDateIndex>, - pub timeindexes_to_open_in_sats: ComputedVecsFromDateIndex>, - pub timeindexes_to_high_in_sats: ComputedVecsFromDateIndex>, - pub timeindexes_to_low_in_sats: ComputedVecsFromDateIndex>, - pub timeindexes_to_close_in_sats: ComputedVecsFromDateIndex>, - pub chainindexes_to_close: ComputedVecsFromHeightStrict>, - pub chainindexes_to_high: ComputedVecsFromHeightStrict>, - pub chainindexes_to_low: ComputedVecsFromHeightStrict>, - pub chainindexes_to_open: ComputedVecsFromHeightStrict>, - pub chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict>, - pub chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict>, - pub chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict>, - pub chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict>, - pub weekindex_to_ohlc: EagerVec, - pub weekindex_to_ohlc_in_sats: EagerVec, - pub difficultyepoch_to_ohlc: EagerVec, - pub difficultyepoch_to_ohlc_in_sats: EagerVec, - pub monthindex_to_ohlc: EagerVec, - pub monthindex_to_ohlc_in_sats: EagerVec, - pub quarterindex_to_ohlc: EagerVec, - pub quarterindex_to_ohlc_in_sats: EagerVec, - pub semesterindex_to_ohlc: EagerVec, - pub semesterindex_to_ohlc_in_sats: EagerVec, - pub yearindex_to_ohlc: EagerVec, - pub yearindex_to_ohlc_in_sats: EagerVec, - // pub halvingepoch_to_ohlc: StorableVec, - // pub halvingepoch_to_ohlc_in_sats: StorableVec, - pub decadeindex_to_ohlc: EagerVec, - pub decadeindex_to_ohlc_in_sats: EagerVec, + file: Arc, + fetcher: Fetcher, + + pub dateindex_to_ohlc_in_cents: RawVec, + pub height_to_ohlc_in_cents: RawVec, } -const VERSION: Version = Version::ZERO; -const VERSION_IN_SATS: Version = Version::ZERO; - impl Vecs { - pub fn forced_import( - file: &Arc, - fetched_file: &Arc, - version: Version, - computation: Computation, - format: Format, - indexes: &indexes::Vecs, - ) -> color_eyre::Result { + pub fn forced_import(parent: &Path, fetcher: Fetcher, version: Version) -> Result { + let file = Arc::new(File::open(&parent.join("fetched"))?); + Ok(Self { - dateindex_to_ohlc_in_cents: EagerVec::forced_import( - fetched_file, + fetcher, + + dateindex_to_ohlc_in_cents: RawVec::forced_import( + &file, "ohlc_in_cents", version + Version::ZERO, - format, )?, - dateindex_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - dateindex_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - dateindex_to_close_in_cents: EagerVec::forced_import( - file, - "close_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - dateindex_to_high_in_cents: EagerVec::forced_import( - file, - "high_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - dateindex_to_low_in_cents: EagerVec::forced_import( - file, - "low_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - dateindex_to_open_in_cents: EagerVec::forced_import( - file, - "open_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - height_to_ohlc_in_cents: EagerVec::forced_import( - fetched_file, + height_to_ohlc_in_cents: RawVec::forced_import( + &file, "ohlc_in_cents", version + Version::ZERO, - format, - )?, - height_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - height_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - height_to_close_in_cents: EagerVec::forced_import( - file, - "close_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - height_to_high_in_cents: EagerVec::forced_import( - file, - "high_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - height_to_low_in_cents: EagerVec::forced_import( - file, - "low_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - height_to_open_in_cents: EagerVec::forced_import( - file, - "open_in_cents", - version + VERSION + Version::ZERO, - format, - )?, - timeindexes_to_open: ComputedVecsFromDateIndex::forced_import( - file, - "open", - Source::Compute, - version + VERSION + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_first(), - )?, - timeindexes_to_high: ComputedVecsFromDateIndex::forced_import( - file, - "high", - Source::Compute, - version + VERSION + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_max(), - )?, - timeindexes_to_low: ComputedVecsFromDateIndex::forced_import( - file, - "low", - Source::Compute, - version + VERSION + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_min(), - )?, - timeindexes_to_close: ComputedVecsFromDateIndex::forced_import( - file, - "close", - Source::Compute, - version + VERSION + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_last(), - )?, - timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import( - file, - "open_in_sats", - Source::Compute, - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_first(), - )?, - timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import( - file, - "high_in_sats", - Source::Compute, - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_max(), - )?, - timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import( - file, - "low_in_sats", - Source::Compute, - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_min(), - )?, - timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import( - file, - "close_in_sats", - Source::Compute, - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - computation, - indexes, - VecBuilderOptions::default().add_last(), - )?, - chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( - file, - "open", - version + VERSION + Version::ZERO, - format, - VecBuilderOptions::default().add_first(), - )?, - chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( - file, - "high", - version + VERSION + Version::ZERO, - format, - VecBuilderOptions::default().add_max(), - )?, - chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( - file, - "low", - version + VERSION + Version::ZERO, - format, - VecBuilderOptions::default().add_min(), - )?, - chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( - file, - "close", - version + VERSION + Version::ZERO, - format, - VecBuilderOptions::default().add_last(), - )?, - chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import( - file, - "open_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - VecBuilderOptions::default().add_first(), - )?, - chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import( - file, - "high_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - VecBuilderOptions::default().add_max(), - )?, - chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import( - file, - "low_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - VecBuilderOptions::default().add_min(), - )?, - chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import( - file, - "close_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - VecBuilderOptions::default().add_last(), - )?, - weekindex_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - weekindex_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - difficultyepoch_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - monthindex_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - monthindex_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - quarterindex_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - quarterindex_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - semesterindex_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - semesterindex_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - yearindex_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - yearindex_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, - )?, - // halvingepoch_to_ohlc: StorableVec::forced_import(file, - // "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?, - decadeindex_to_ohlc: EagerVec::forced_import( - file, - "ohlc", - version + VERSION + Version::ZERO, - format, - )?, - decadeindex_to_ohlc_in_sats: EagerVec::forced_import( - file, - "ohlc_in_sats", - version + VERSION + VERSION_IN_SATS + Version::ZERO, - format, )?, + + file, }) } @@ -398,78 +47,49 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, starting_indexes: &Indexes, - fetcher: &mut Fetcher, exit: &Exit, - ) -> color_eyre::Result<()> { - let mut height_to_timestamp_iter = indexer.vecs.height_to_timestamp.iter(); - self.height_to_ohlc_in_cents.compute_transform( - starting_indexes.height, - &indexer.vecs.height_to_timestamp, - |(h, t, ..)| { - let ohlc = fetcher - .get_height( - h, - t, - h.decremented() - .map(|prev_h| height_to_timestamp_iter.unwrap_get_inner(prev_h)), - ) - .unwrap(); - (h, ohlc) - }, - exit, - )?; - - self.height_to_open_in_cents.compute_transform( - starting_indexes.height, - &self.height_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.open), - exit, - )?; - - self.height_to_high_in_cents.compute_transform( - starting_indexes.height, - &self.height_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.high), - exit, - )?; - - self.height_to_low_in_cents.compute_transform( - starting_indexes.height, - &self.height_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.low), - exit, - )?; - - self.height_to_close_in_cents.compute_transform( - starting_indexes.height, - &self.height_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.close), - exit, - )?; - - self.height_to_ohlc.compute_transform( - starting_indexes.height, - &self.height_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), - exit, - )?; + ) -> Result<()> { + let height_to_timestamp = &indexer.vecs.height_to_timestamp; + height_to_timestamp + .iter_at(starting_indexes.height) + .try_for_each(|(i, v)| -> Result<()> { + let v = v.into_owned(); + self.height_to_ohlc_in_cents.forced_push_at( + i, + self.fetcher + .get_height( + i, + v, + i.decremented().map(|prev_i| { + height_to_timestamp.into_iter().unwrap_get_inner(prev_i) + }), + ) + .unwrap(), + exit, + )?; + Ok(()) + })?; + self.height_to_ohlc_in_cents.safe_flush(exit)?; let mut prev = None; - self.dateindex_to_ohlc_in_cents.compute_transform( - starting_indexes.dateindex, - &indexes.dateindex_to_date, - |(di, d, this)| { + indexes + .dateindex_to_date + .iter_at(starting_indexes.dateindex) + .try_for_each(|(i, v)| -> Result<()> { + let d = v.into_owned(); if prev.is_none() { - let i = di.unwrap_to_usize(); + let i = i.unwrap_to_usize(); prev.replace(if i > 0 { - this.into_iter().unwrap_get_inner_(i - 1) + self.dateindex_to_ohlc_in_cents + .into_iter() + .unwrap_get_inner_(i - 1) } else { OHLCCents::default() }); } - let ohlc = if di.unwrap_to_usize() + 100 >= this.len() - && let Ok(mut ohlc) = fetcher.get_date(d) + let ohlc = if i.unwrap_to_usize() + 100 >= self.dateindex_to_ohlc_in_cents.len() + && let Ok(mut ohlc) = self.fetcher.get_date(d) { let prev_open = *prev.as_ref().unwrap().close; *ohlc.open = prev_open; @@ -482,788 +102,22 @@ impl Vecs { prev.replace(ohlc.clone()); - (di, ohlc) - }, - exit, - )?; + self.dateindex_to_ohlc_in_cents + .forced_push_at(i, ohlc, exit)?; - self.dateindex_to_open_in_cents.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.open), - exit, - )?; - - self.dateindex_to_high_in_cents.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.high), - exit, - )?; - - self.dateindex_to_low_in_cents.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.low), - exit, - )?; - - self.dateindex_to_close_in_cents.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, ohlc.close), - exit, - )?; - - self.dateindex_to_ohlc.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc_in_cents, - |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), - exit, - )?; - - self.timeindexes_to_close.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.close), - exit, - ) - }, - )?; - - self.timeindexes_to_high.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.high), - exit, - ) - }, - )?; - - self.timeindexes_to_low.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.low), - exit, - ) - }, - )?; - - self.timeindexes_to_open.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - &self.dateindex_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.open), - exit, - ) - }, - )?; - - self.chainindexes_to_close.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.height_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.close), - exit, - ) - }, - )?; - - self.chainindexes_to_high.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.height_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.high), - exit, - ) - }, - )?; - - self.chainindexes_to_low.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.height_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.low), - exit, - ) - }, - )?; - - self.chainindexes_to_open.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.height_to_ohlc, - |(di, ohlc, ..)| (di, ohlc.open), - exit, - ) - }, - )?; - - let mut weekindex_first_iter = self.timeindexes_to_open.weekindex.unwrap_first().iter(); - let mut weekindex_max_iter = self.timeindexes_to_high.weekindex.unwrap_max().iter(); - let mut weekindex_min_iter = self.timeindexes_to_low.weekindex.unwrap_min().iter(); - self.weekindex_to_ohlc.compute_transform( - starting_indexes.weekindex, - self.timeindexes_to_close.weekindex.unwrap_last(), - |(i, close, ..)| { - let open = weekindex_first_iter.unwrap_get_inner(i); - let high = weekindex_max_iter.unwrap_get_inner(i); - let low = weekindex_min_iter.unwrap_get_inner(i); - ( - i, - OHLCDollars { - open, - high, - low, - close, - }, - ) - }, - exit, - )?; - - let mut difficultyepoch_first_iter = self - .chainindexes_to_open - .difficultyepoch - .unwrap_first() - .iter(); - let mut difficultyepoch_max_iter = self - .chainindexes_to_high - .difficultyepoch - .unwrap_max() - .iter(); - let mut difficultyepoch_min_iter = - self.chainindexes_to_low.difficultyepoch.unwrap_min().iter(); - self.difficultyepoch_to_ohlc.compute_transform( - starting_indexes.difficultyepoch, - self.chainindexes_to_close.difficultyepoch.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCDollars { - open: difficultyepoch_first_iter.unwrap_get_inner(i), - high: difficultyepoch_max_iter.unwrap_get_inner(i), - low: difficultyepoch_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut monthindex_first_iter = self.timeindexes_to_open.monthindex.unwrap_first().iter(); - let mut monthindex_max_iter = self.timeindexes_to_high.monthindex.unwrap_max().iter(); - let mut monthindex_min_iter = self.timeindexes_to_low.monthindex.unwrap_min().iter(); - self.monthindex_to_ohlc.compute_transform( - starting_indexes.monthindex, - self.timeindexes_to_close.monthindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCDollars { - open: monthindex_first_iter.unwrap_get_inner(i), - high: monthindex_max_iter.unwrap_get_inner(i), - low: monthindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut quarterindex_first_iter = - self.timeindexes_to_open.quarterindex.unwrap_first().iter(); - let mut quarterindex_max_iter = self.timeindexes_to_high.quarterindex.unwrap_max().iter(); - let mut quarterindex_min_iter = self.timeindexes_to_low.quarterindex.unwrap_min().iter(); - self.quarterindex_to_ohlc.compute_transform( - starting_indexes.quarterindex, - self.timeindexes_to_close.quarterindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCDollars { - open: quarterindex_first_iter.unwrap_get_inner(i), - high: quarterindex_max_iter.unwrap_get_inner(i), - low: quarterindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut semesterindex_first_iter = - self.timeindexes_to_open.semesterindex.unwrap_first().iter(); - let mut semesterindex_max_iter = self.timeindexes_to_high.semesterindex.unwrap_max().iter(); - let mut semesterindex_min_iter = self.timeindexes_to_low.semesterindex.unwrap_min().iter(); - self.semesterindex_to_ohlc.compute_transform( - starting_indexes.semesterindex, - self.timeindexes_to_close.semesterindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCDollars { - open: semesterindex_first_iter.unwrap_get_inner(i), - high: semesterindex_max_iter.unwrap_get_inner(i), - low: semesterindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut yearindex_first_iter = self.timeindexes_to_open.yearindex.unwrap_first().iter(); - let mut yearindex_max_iter = self.timeindexes_to_high.yearindex.unwrap_max().iter(); - let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter(); - self.yearindex_to_ohlc.compute_transform( - starting_indexes.yearindex, - self.timeindexes_to_close.yearindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCDollars { - open: yearindex_first_iter.unwrap_get_inner(i), - high: yearindex_max_iter.unwrap_get_inner(i), - low: yearindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - // self.halvingepoch_to_ohlc - // .compute_transform(starting_indexes.halvingepoch, other, t, exit)?; - - let mut decadeindex_first_iter = self.timeindexes_to_open.decadeindex.unwrap_first().iter(); - let mut decadeindex_max_iter = self.timeindexes_to_high.decadeindex.unwrap_max().iter(); - let mut decadeindex_min_iter = self.timeindexes_to_low.decadeindex.unwrap_min().iter(); - self.decadeindex_to_ohlc.compute_transform( - starting_indexes.decadeindex, - self.timeindexes_to_close.decadeindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCDollars { - open: decadeindex_first_iter.unwrap_get_inner(i), - high: decadeindex_max_iter.unwrap_get_inner(i), - low: decadeindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - self.chainindexes_to_open_in_sats.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.chainindexes_to_open.height, - |(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)), - exit, - ) - }, - )?; - - self.chainindexes_to_high_in_sats.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.chainindexes_to_low.height, - |(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)), - exit, - ) - }, - )?; - - self.chainindexes_to_low_in_sats.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.chainindexes_to_high.height, - |(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)), - exit, - ) - }, - )?; - - self.chainindexes_to_close_in_sats.compute( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.height, - &self.chainindexes_to_close.height, - |(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)), - exit, - ) - }, - )?; - - self.timeindexes_to_open_in_sats.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - self.timeindexes_to_open.dateindex.as_ref().unwrap(), - |(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)), - exit, - ) - }, - )?; - - self.timeindexes_to_high_in_sats.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - self.timeindexes_to_low.dateindex.as_ref().unwrap(), - |(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)), - exit, - ) - }, - )?; - - self.timeindexes_to_low_in_sats.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - self.timeindexes_to_high.dateindex.as_ref().unwrap(), - |(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)), - exit, - ) - }, - )?; - - self.timeindexes_to_close_in_sats.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_transform( - starting_indexes.dateindex, - self.timeindexes_to_close.dateindex.as_ref().unwrap(), - |(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)), - exit, - ) - }, - )?; - - let mut height_first_iter = self.chainindexes_to_open_in_sats.height.iter(); - let mut height_max_iter = self.chainindexes_to_high_in_sats.height.iter(); - let mut height_min_iter = self.chainindexes_to_low_in_sats.height.iter(); - self.height_to_ohlc_in_sats.compute_transform( - starting_indexes.height, - &self.chainindexes_to_close_in_sats.height, - |(i, close, ..)| { - ( - i, - OHLCSats { - open: height_first_iter.unwrap_get_inner(i), - high: height_max_iter.unwrap_get_inner(i), - low: height_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut dateindex_first_iter = self - .timeindexes_to_open_in_sats - .dateindex - .as_ref() - .unwrap() - .iter(); - let mut dateindex_max_iter = self - .timeindexes_to_high_in_sats - .dateindex - .as_ref() - .unwrap() - .iter(); - let mut dateindex_min_iter = self - .timeindexes_to_low_in_sats - .dateindex - .as_ref() - .unwrap() - .iter(); - self.dateindex_to_ohlc_in_sats.compute_transform( - starting_indexes.dateindex, - self.timeindexes_to_close_in_sats - .dateindex - .as_ref() - .unwrap(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: dateindex_first_iter.unwrap_get_inner(i), - high: dateindex_max_iter.unwrap_get_inner(i), - low: dateindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut weekindex_first_iter = self - .timeindexes_to_open_in_sats - .weekindex - .unwrap_first() - .iter(); - let mut weekindex_max_iter = self - .timeindexes_to_high_in_sats - .weekindex - .unwrap_max() - .iter(); - let mut weekindex_min_iter = self - .timeindexes_to_low_in_sats - .weekindex - .unwrap_min() - .iter(); - self.weekindex_to_ohlc_in_sats.compute_transform( - starting_indexes.weekindex, - self.timeindexes_to_close_in_sats.weekindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: weekindex_first_iter.unwrap_get_inner(i), - high: weekindex_max_iter.unwrap_get_inner(i), - low: weekindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut difficultyepoch_first_iter = self - .chainindexes_to_open_in_sats - .difficultyepoch - .unwrap_first() - .iter(); - let mut difficultyepoch_max_iter = self - .chainindexes_to_high_in_sats - .difficultyepoch - .unwrap_max() - .iter(); - let mut difficultyepoch_min_iter = self - .chainindexes_to_low_in_sats - .difficultyepoch - .unwrap_min() - .iter(); - self.difficultyepoch_to_ohlc_in_sats.compute_transform( - starting_indexes.difficultyepoch, - self.chainindexes_to_close_in_sats - .difficultyepoch - .unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: difficultyepoch_first_iter.unwrap_get_inner(i), - high: difficultyepoch_max_iter.unwrap_get_inner(i), - low: difficultyepoch_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut monthindex_first_iter = self - .timeindexes_to_open_in_sats - .monthindex - .unwrap_first() - .iter(); - let mut monthindex_max_iter = self - .timeindexes_to_high_in_sats - .monthindex - .unwrap_max() - .iter(); - let mut monthindex_min_iter = self - .timeindexes_to_low_in_sats - .monthindex - .unwrap_min() - .iter(); - self.monthindex_to_ohlc_in_sats.compute_transform( - starting_indexes.monthindex, - self.timeindexes_to_close_in_sats.monthindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: monthindex_first_iter.unwrap_get_inner(i), - high: monthindex_max_iter.unwrap_get_inner(i), - low: monthindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut quarterindex_first_iter = self - .timeindexes_to_open_in_sats - .quarterindex - .unwrap_first() - .iter(); - let mut quarterindex_max_iter = self - .timeindexes_to_high_in_sats - .quarterindex - .unwrap_max() - .iter(); - let mut quarterindex_min_iter = self - .timeindexes_to_low_in_sats - .quarterindex - .unwrap_min() - .iter(); - self.quarterindex_to_ohlc_in_sats.compute_transform( - starting_indexes.quarterindex, - self.timeindexes_to_close_in_sats.quarterindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: quarterindex_first_iter.unwrap_get_inner(i), - high: quarterindex_max_iter.unwrap_get_inner(i), - low: quarterindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut semesterindex_first_iter = self - .timeindexes_to_open_in_sats - .semesterindex - .unwrap_first() - .iter(); - let mut semesterindex_max_iter = self - .timeindexes_to_high_in_sats - .semesterindex - .unwrap_max() - .iter(); - let mut semesterindex_min_iter = self - .timeindexes_to_low_in_sats - .semesterindex - .unwrap_min() - .iter(); - self.semesterindex_to_ohlc_in_sats.compute_transform( - starting_indexes.semesterindex, - self.timeindexes_to_close_in_sats - .semesterindex - .unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: semesterindex_first_iter.unwrap_get_inner(i), - high: semesterindex_max_iter.unwrap_get_inner(i), - low: semesterindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - let mut yearindex_first_iter = self - .timeindexes_to_open_in_sats - .yearindex - .unwrap_first() - .iter(); - let mut yearindex_max_iter = self - .timeindexes_to_high_in_sats - .yearindex - .unwrap_max() - .iter(); - let mut yearindex_min_iter = self - .timeindexes_to_low_in_sats - .yearindex - .unwrap_min() - .iter(); - self.yearindex_to_ohlc_in_sats.compute_transform( - starting_indexes.yearindex, - self.timeindexes_to_close_in_sats.yearindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: yearindex_first_iter.unwrap_get_inner(i), - high: yearindex_max_iter.unwrap_get_inner(i), - low: yearindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; - - // self.halvingepoch_to_ohlc - // _in_sats.compute_transform(starting_indexes.halvingepoch, other, t, exit)?; - - let mut decadeindex_first_iter = self - .timeindexes_to_open_in_sats - .decadeindex - .unwrap_first() - .iter(); - let mut decadeindex_max_iter = self - .timeindexes_to_high_in_sats - .decadeindex - .unwrap_max() - .iter(); - let mut decadeindex_min_iter = self - .timeindexes_to_low_in_sats - .decadeindex - .unwrap_min() - .iter(); - self.decadeindex_to_ohlc_in_sats.compute_transform( - starting_indexes.decadeindex, - self.timeindexes_to_close_in_sats.decadeindex.unwrap_last(), - |(i, close, ..)| { - ( - i, - OHLCSats { - open: decadeindex_first_iter.unwrap_get_inner(i), - high: decadeindex_max_iter.unwrap_get_inner(i), - low: decadeindex_min_iter.unwrap_get_inner(i), - close, - }, - ) - }, - exit, - )?; + Ok(()) + })?; + self.dateindex_to_ohlc_in_cents.safe_flush(exit)?; + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { vec![ - vec![ - &self.dateindex_to_close_in_cents as &dyn AnyCollectableVec, - &self.dateindex_to_high_in_cents, - &self.dateindex_to_low_in_cents, - &self.dateindex_to_ohlc, - &self.dateindex_to_ohlc_in_cents, - &self.dateindex_to_open_in_cents, - &self.height_to_close_in_cents, - &self.height_to_high_in_cents, - &self.height_to_low_in_cents, - &self.height_to_ohlc, - &self.height_to_ohlc_in_cents, - &self.height_to_open_in_cents, - &self.weekindex_to_ohlc, - &self.difficultyepoch_to_ohlc, - &self.monthindex_to_ohlc, - &self.quarterindex_to_ohlc, - &self.semesterindex_to_ohlc, - &self.yearindex_to_ohlc, - // &self.halvingepoch_to_ohlc, - &self.decadeindex_to_ohlc, - &self.height_to_ohlc_in_sats, - &self.dateindex_to_ohlc_in_sats, - &self.weekindex_to_ohlc_in_sats, - &self.difficultyepoch_to_ohlc_in_sats, - &self.monthindex_to_ohlc_in_sats, - &self.quarterindex_to_ohlc_in_sats, - &self.semesterindex_to_ohlc_in_sats, - &self.yearindex_to_ohlc_in_sats, - // &self.halvingepoch_to_ohlc_in_sats, - &self.decadeindex_to_ohlc_in_sats, - ], - self.timeindexes_to_close.vecs(), - self.timeindexes_to_high.vecs(), - self.timeindexes_to_low.vecs(), - self.timeindexes_to_open.vecs(), - self.chainindexes_to_close.vecs(), - self.chainindexes_to_high.vecs(), - self.chainindexes_to_low.vecs(), - self.chainindexes_to_open.vecs(), - self.timeindexes_to_close_in_sats.vecs(), - self.timeindexes_to_high_in_sats.vecs(), - self.timeindexes_to_low_in_sats.vecs(), - self.timeindexes_to_open_in_sats.vecs(), - self.chainindexes_to_close_in_sats.vecs(), - self.chainindexes_to_high_in_sats.vecs(), - self.chainindexes_to_low_in_sats.vecs(), - self.chainindexes_to_open_in_sats.vecs(), + &self.dateindex_to_ohlc_in_cents as &dyn AnyCollectableVec, + &self.height_to_ohlc_in_cents, ] - .into_iter() - .flatten() - .collect::>() } } diff --git a/crates/brk_computer/src/grouped/builder_computed.rs b/crates/brk_computer/src/grouped/builder_computed.rs index 627fa177a..6b492f9d1 100644 --- a/crates/brk_computer/src/grouped/builder_computed.rs +++ b/crates/brk_computer/src/grouped/builder_computed.rs @@ -1,10 +1,11 @@ use std::sync::Arc; -use brk_core::{FromCoarserIndex, Result, Version}; -use brk_exit::Exit; +use brk_error::Result; + +use brk_structs::Version; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation, - ComputedVec, ComputedVecFrom2, File, Format, StoredIndex, + AnyBoxedIterableVec, AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, + ComputedVec, ComputedVecFrom2, Exit, File, Format, FromCoarserIndex, StoredIndex, }; use crate::grouped::{EagerVecBuilder, VecBuilderOptions}; @@ -44,11 +45,11 @@ where version: Version, format: Format, computation: Computation, - source: Option>, + source: Option>, source_extra: &EagerVecBuilder, - len_source: BoxedAnyIterableVec, + len_source: AnyBoxedIterableVec, options: ComputedVecBuilderOptions, - ) -> color_eyre::Result { + ) -> Result { let only_one_active = options.is_only_one_active(); let suffix = |s: &str| format!("{name}_{s}"); diff --git a/crates/brk_computer/src/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs index 57001e54c..3ff073125 100644 --- a/crates/brk_computer/src/grouped/builder_eager.rs +++ b/crates/brk_computer/src/grouped/builder_eager.rs @@ -1,11 +1,11 @@ use std::sync::Arc; -use brk_core::{CheckedSub, Result, StoredUsize, Version}; -use brk_exit::Exit; +use brk_error::{Error, Result}; +use brk_structs::{CheckedSub, StoredU64, Version}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyVec, EagerVec, File, Format, StoredIndex, StoredType, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, File, Format, + GenericStoredVec, StoredIndex, StoredRaw, }; -use color_eyre::eyre::ContextCompat; use crate::utils::get_percentile; @@ -44,7 +44,7 @@ where version: Version, format: Format, options: VecBuilderOptions, - ) -> color_eyre::Result { + ) -> Result { let only_one_active = options.is_only_one_active(); let suffix = |s: &str| format!("{name}_{s}"); @@ -213,8 +213,9 @@ where cumulative_vec.iter().unwrap_get_inner(index) }); source.iter_at(index).try_for_each(|(i, v)| -> Result<()> { - cumulative = cumulative.clone() + v.into_owned(); - cumulative_vec.forced_push_at(i, cumulative.clone(), exit) + cumulative += v.into_owned(); + cumulative_vec.forced_push_at(i, cumulative, exit)?; + Ok(()) })?; self.safe_flush(exit)?; @@ -227,11 +228,11 @@ where max_from: I, source: &impl AnyIterableVec, first_indexes: &impl AnyIterableVec, - count_indexes: &impl AnyIterableVec, + count_indexes: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where - I2: StoredIndex + StoredType + CheckedSub, + I2: StoredIndex + StoredRaw + CheckedSub, { self.validate_computed_version_or_reset_file( source.version() + first_indexes.version() + count_indexes.version(), @@ -265,7 +266,7 @@ where } if let Some(last) = self.last.as_mut() { - let count_index = *count_index; + let count_index = *count_index as usize; if count_index == 0 { panic!("should compute last if count can be 0") } @@ -295,7 +296,7 @@ where if needs_values { source_iter.set(first_index); let mut values = (&mut source_iter) - .take(*count_index) + .take(*count_index as usize) .map(|(_, v)| v.into_owned()) .collect::>(); @@ -305,9 +306,9 @@ where if let Some(max) = self.max.as_mut() { max.forced_push_at( i, - values + *values .last() - .context("expect some") + .ok_or(Error::Str("expect some")) .inspect_err(|_| { dbg!( &values, @@ -320,8 +321,7 @@ where source.name() ); }) - .unwrap() - .clone(), + .unwrap(), exit, )?; } @@ -347,7 +347,7 @@ where } if let Some(min) = self.min.as_mut() { - min.forced_push_at(i, values.first().unwrap().clone(), exit)?; + min.forced_push_at(i, *values.first().unwrap(), exit)?; } } @@ -356,18 +356,18 @@ where let sum = values.into_iter().fold(T::from(0), |a, b| a + b); if let Some(average) = self.average.as_mut() { - let avg = sum.clone() / len; + let avg = sum / len; average.forced_push_at(i, avg, exit)?; } if needs_sum_or_cumulative { if let Some(sum_vec) = self.sum.as_mut() { - sum_vec.forced_push_at(i, sum.clone(), exit)?; + sum_vec.forced_push_at(i, sum, exit)?; } if let Some(cumulative_vec) = self.cumulative.as_mut() { - let t = cumulative.as_ref().unwrap().clone() + sum; - cumulative.replace(t.clone()); + let t = cumulative.unwrap() + sum; + cumulative.replace(t); cumulative_vec.forced_push_at(i, t, exit)?; } } @@ -388,11 +388,11 @@ where max_from: I, source: &EagerVecBuilder, first_indexes: &impl AnyIterableVec, - count_indexes: &impl AnyIterableVec, + count_indexes: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where - I2: StoredIndex + StoredType + CheckedSub, + I2: StoredIndex + StoredRaw + CheckedSub, { if self._90p.is_some() || self._75p.is_some() @@ -440,7 +440,7 @@ where } if let Some(last) = self.last.as_mut() { - let count_index = *count_index; + let count_index = *count_index as usize; if count_index == 0 { panic!("should compute last if count can be 0") } @@ -464,22 +464,22 @@ where let source_max_iter = source_max_iter.as_mut().unwrap(); source_max_iter.set(first_index); let mut values = source_max_iter - .take(*count_index) + .take(*count_index as usize) .map(|(_, v)| v.into_owned()) .collect::>(); values.sort_unstable(); - max.forced_push_at(i, values.last().unwrap().clone(), exit)?; + max.forced_push_at(i, *values.last().unwrap(), exit)?; } if let Some(min) = self.min.as_mut() { let source_min_iter = source_min_iter.as_mut().unwrap(); source_min_iter.set(first_index); let mut values = source_min_iter - .take(*count_index) + .take(*count_index as usize) .map(|(_, v)| v.into_owned()) .collect::>(); values.sort_unstable(); - min.forced_push_at(i, values.first().unwrap().clone(), exit)?; + min.forced_push_at(i, *values.first().unwrap(), exit)?; } } @@ -488,7 +488,7 @@ where let source_average_iter = source_average_iter.as_mut().unwrap(); source_average_iter.set(first_index); let values = source_average_iter - .take(*count_index) + .take(*count_index as usize) .map(|(_, v)| v.into_owned()) .collect::>(); @@ -504,19 +504,19 @@ where let source_sum_iter = source_sum_iter.as_mut().unwrap(); source_sum_iter.set(first_index); let values = source_sum_iter - .take(*count_index) + .take(*count_index as usize) .map(|(_, v)| v.into_owned()) .collect::>(); let sum = values.into_iter().fold(T::from(0), |a, b| a + b); if let Some(sum_vec) = self.sum.as_mut() { - sum_vec.forced_push_at(i, sum.clone(), exit)?; + sum_vec.forced_push_at(i, sum, exit)?; } if let Some(cumulative_vec) = self.cumulative.as_mut() { - let t = cumulative.as_ref().unwrap().clone() + sum; - cumulative.replace(t.clone()); + let t = cumulative.unwrap() + sum; + cumulative.replace(t); cumulative_vec.forced_push_at(i, t, exit)?; } } diff --git a/crates/brk_computer/src/grouped/computed.rs b/crates/brk_computer/src/grouped/computed.rs new file mode 100644 index 000000000..849e61b59 --- /dev/null +++ b/crates/brk_computer/src/grouped/computed.rs @@ -0,0 +1,23 @@ +use std::ops::{Add, AddAssign, Div}; + +use brk_vecs::StoredCompressed; + +pub trait ComputedType +where + Self: StoredCompressed + + From + + Div + + Add + + AddAssign + + Ord, +{ +} +impl ComputedType for T where + T: StoredCompressed + + From + + Div + + Add + + AddAssign + + Ord +{ +} diff --git a/crates/brk_computer/src/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs index f4ca684dc..654c6ca6a 100644 --- a/crates/brk_computer/src/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -1,13 +1,14 @@ use std::sync::Arc; -use brk_core::{ - DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex, - YearIndex, -}; -use brk_exit::Exit; +use brk_error::Result; + use brk_indexer::Indexer; +use brk_structs::{ + DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, +}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format, + AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File, + Format, }; use crate::{Indexes, grouped::ComputedVecBuilder, indexes}; @@ -45,7 +46,7 @@ where computation: Computation, indexes: &indexes::Vecs, options: VecBuilderOptions, - ) -> color_eyre::Result { + ) -> Result { let dateindex = source.is_compute().then(|| { EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap() }); @@ -141,7 +142,7 @@ where starting_indexes: &Indexes, exit: &Exit, mut compute: F, - ) -> color_eyre::Result<()> + ) -> Result<()> where F: FnMut( &mut EagerVec, @@ -169,7 +170,7 @@ where starting_indexes: &Indexes, exit: &Exit, dateindex: Option<&impl AnyIterableVec>, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if let Some(dateindex) = dateindex { self.dateindex_extra .extend(starting_indexes.dateindex, dateindex, exit)?; diff --git a/crates/brk_computer/src/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs index 3bbe1e7d3..17be32533 100644 --- a/crates/brk_computer/src/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -1,13 +1,15 @@ use std::sync::Arc; -use brk_core::{ - DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, - SemesterIndex, Version, WeekIndex, YearIndex, -}; -use brk_exit::Exit; +use brk_error::Result; + use brk_indexer::Indexer; +use brk_structs::{ + DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex, + Version, WeekIndex, YearIndex, +}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format, + AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File, + Format, }; use crate::{ @@ -53,7 +55,7 @@ where computation: Computation, indexes: &indexes::Vecs, options: VecBuilderOptions, - ) -> color_eyre::Result { + ) -> Result { let height = source.is_compute().then(|| { EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap() }); @@ -164,7 +166,7 @@ where starting_indexes: &Indexes, exit: &Exit, mut compute: F, - ) -> color_eyre::Result<()> + ) -> Result<()> where F: FnMut(&mut EagerVec, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>, { @@ -186,7 +188,7 @@ where starting_indexes: &Indexes, exit: &Exit, height_vec: Option<&impl AnyIterableVec>, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if let Some(height) = height_vec { self.height_extra .extend(starting_indexes.height, height, exit)?; diff --git a/crates/brk_computer/src/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs index d59ca609c..4fe6819df 100644 --- a/crates/brk_computer/src/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -1,9 +1,10 @@ use std::sync::Arc; -use brk_core::{DifficultyEpoch, Height, Result, Version}; -use brk_exit::Exit; +use brk_error::Result; + use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, EagerVec, File, Format}; +use brk_structs::{DifficultyEpoch, Height, Version}; +use brk_vecs::{AnyCollectableVec, EagerVec, Exit, File, Format}; use crate::{Indexes, indexes}; @@ -33,7 +34,7 @@ where version: Version, format: Format, options: VecBuilderOptions, - ) -> color_eyre::Result { + ) -> Result { let height = EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?; @@ -68,7 +69,7 @@ where starting_indexes: &Indexes, exit: &Exit, mut compute: F, - ) -> color_eyre::Result<()> + ) -> Result<()> where F: FnMut(&mut EagerVec, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>, { diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index c346d101d..3495a643c 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -1,20 +1,20 @@ use std::sync::Arc; -use brk_core::{ - Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex, - Result, Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex, -}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; +use brk_structs::{ + Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex, + Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex, +}; use brk_vecs::{ - AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec, - File, Format, StoredIndex, VecIterator, + AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, + Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator, }; use crate::{ - Indexes, fetched, + Indexes, grouped::{ComputedVecBuilder, Source}, - indexes, + indexes, price, }; use super::{ComputedType, EagerVecBuilder, VecBuilderOptions}; @@ -54,7 +54,7 @@ where computation: Computation, indexes: &indexes::Vecs, options: VecBuilderOptions, - ) -> color_eyre::Result { + ) -> Result { let txindex = source.is_compute().then(|| { Box::new( EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format) @@ -170,7 +170,7 @@ where // starting_indexes: &Indexes, // exit: &Exit, // mut compute: F, - // ) -> color_eyre::Result<()> + // ) -> Result<()> // where // F: FnMut( // &mut EagerVec, @@ -495,7 +495,7 @@ impl ComputedVecsFromTxindex { exit: &Exit, bitcoin: &ComputedVecsFromTxindex, txindex: Option<&impl CollectableVec>, - fetched: &fetched::Vecs, + price: &price::Vecs, ) -> Result<()> { let txindex_version = if let Some(txindex) = txindex { txindex.version() @@ -508,7 +508,7 @@ impl ComputedVecsFromTxindex { let starting_index = self.height.starting_index(starting_indexes.height); - let mut close_iter = fetched.chainindexes_to_close.height.into_iter(); + let mut close_iter = price.chainindexes_to_close.height.into_iter(); (starting_index.unwrap_to_usize()..indexer.vecs.height_to_weight.len()) .map(Height::from) diff --git a/crates/brk_computer/src/grouped/mod.rs b/crates/brk_computer/src/grouped/mod.rs index 5c4e67968..3970c3246 100644 --- a/crates/brk_computer/src/grouped/mod.rs +++ b/crates/brk_computer/src/grouped/mod.rs @@ -1,12 +1,12 @@ mod builder_computed; mod builder_eager; +mod computed; mod from_dateindex; mod from_height; mod from_height_strict; mod from_txindex; mod ratio_from_dateindex; mod source; -mod r#type; mod value_from_dateindex; mod value_from_height; mod value_from_txindex; @@ -14,13 +14,13 @@ mod value_height; pub use builder_computed::*; pub use builder_eager::*; +use computed::*; pub use from_dateindex::*; pub use from_height::*; pub use from_height_strict::*; pub use from_txindex::*; pub use ratio_from_dateindex::*; pub use source::*; -use r#type::*; pub use value_from_dateindex::*; pub use value_from_height::*; pub use value_from_txindex::*; diff --git a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs index 53eee2292..bfd12e32a 100644 --- a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs @@ -1,14 +1,14 @@ use std::{f32, sync::Arc}; -use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; +use brk_structs::{Date, DateIndex, Dollars, StoredF32, Version}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format, - StoredIndex, VecIterator, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, + Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator, }; -use crate::{Indexes, fetched, grouped::source::Source, indexes, utils::get_percentile}; +use crate::{Indexes, grouped::source::Source, indexes, price, utils::get_percentile}; use super::{ComputedVecsFromDateIndex, VecBuilderOptions}; @@ -68,7 +68,7 @@ impl ComputedRatioVecsFromDateIndex { computation: Computation, indexes: &indexes::Vecs, extended: bool, - ) -> color_eyre::Result { + ) -> Result { let options = VecBuilderOptions::default().add_last(); Ok(Self { @@ -570,11 +570,11 @@ impl ComputedRatioVecsFromDateIndex { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: &fetched::Vecs, + price: &price::Vecs, starting_indexes: &Indexes, exit: &Exit, compute: F, - ) -> color_eyre::Result<()> + ) -> Result<()> where F: FnMut( &mut EagerVec, @@ -596,7 +596,7 @@ impl ComputedRatioVecsFromDateIndex { self.compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, date_to_price_opt, @@ -607,16 +607,16 @@ impl ComputedRatioVecsFromDateIndex { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: &fetched::Vecs, + price: &price::Vecs, starting_indexes: &Indexes, exit: &Exit, date_to_price_opt: Option<&impl AnyIterableVec>, - ) -> color_eyre::Result<()> { + ) -> Result<()> { let date_to_price = date_to_price_opt.unwrap_or_else(|| unsafe { std::mem::transmute(&self.price.as_ref().unwrap().dateindex) }); - let closes = fetched.timeindexes_to_close.dateindex.as_ref().unwrap(); + let closes = price.timeindexes_to_close.dateindex.as_ref().unwrap(); self.ratio.compute_all( indexer, @@ -637,7 +637,8 @@ impl ComputedRatioVecsFromDateIndex { } }, exit, - ) + )?; + Ok(()) }, )?; @@ -659,7 +660,8 @@ impl ComputedRatioVecsFromDateIndex { usize::MAX, exit, Some(min_ratio_date), - ) + )?; + Ok(()) }, )?; @@ -675,7 +677,8 @@ impl ComputedRatioVecsFromDateIndex { 7, exit, Some(min_ratio_date), - ) + )?; + Ok(()) }, )?; @@ -691,7 +694,8 @@ impl ComputedRatioVecsFromDateIndex { 30, exit, Some(min_ratio_date), - ) + )?; + Ok(()) }, )?; @@ -707,7 +711,8 @@ impl ComputedRatioVecsFromDateIndex { 365, exit, Some(min_ratio_date), - ) + )?; + Ok(()) }, )?; @@ -723,7 +728,8 @@ impl ComputedRatioVecsFromDateIndex { 4 * 365, exit, Some(min_ratio_date), - ) + )?; + Ok(()) }, )?; @@ -756,7 +762,8 @@ impl ComputedRatioVecsFromDateIndex { ) }, exit, - ) + )?; + Ok(()) }, )?; @@ -766,7 +773,8 @@ impl ComputedRatioVecsFromDateIndex { .try_for_each(|v| -> Result<()> { v.validate_computed_version_or_reset_file( Version::ZERO + v.inner_version() + ratio_version, - ) + )?; + Ok(()) })?; let starting_dateindex = self @@ -1194,7 +1202,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1220,7 +1229,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1246,7 +1256,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1272,7 +1283,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1298,7 +1310,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1324,7 +1337,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1350,7 +1364,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1376,7 +1391,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1402,7 +1418,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1428,7 +1445,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1454,7 +1472,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1480,7 +1499,8 @@ impl ComputedRatioVecsFromDateIndex { (i, price * multiplier) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1496,7 +1516,8 @@ impl ComputedRatioVecsFromDateIndex { self.ratio_sma.as_ref().unwrap().dateindex.as_ref().unwrap(), self.ratio_sd.as_ref().unwrap().dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -1522,7 +1543,8 @@ impl ComputedRatioVecsFromDateIndex { .as_ref() .unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -1548,7 +1570,8 @@ impl ComputedRatioVecsFromDateIndex { .as_ref() .unwrap(), exit, - ) + )?; + Ok(()) }, )?; diff --git a/crates/brk_computer/src/grouped/source.rs b/crates/brk_computer/src/grouped/source.rs index aade698fc..b1a621aa0 100644 --- a/crates/brk_computer/src/grouped/source.rs +++ b/crates/brk_computer/src/grouped/source.rs @@ -1,10 +1,10 @@ -use brk_vecs::BoxedAnyIterableVec; +use brk_vecs::AnyBoxedIterableVec; #[derive(Clone)] pub enum Source { Compute, None, - Vec(BoxedAnyIterableVec), + Vec(AnyBoxedIterableVec), } impl Source { @@ -20,7 +20,7 @@ impl Source { matches!(self, Self::Vec(_)) } - pub fn vec(self) -> Option> { + pub fn vec(self) -> Option> { match self { Self::Vec(v) => Some(v), _ => None, @@ -34,14 +34,14 @@ impl From for Source { } } -impl From> for Source { - fn from(value: BoxedAnyIterableVec) -> Self { +impl From> for Source { + fn from(value: AnyBoxedIterableVec) -> Self { Self::Vec(value) } } -impl From>> for Source { - fn from(value: Option>) -> Self { +impl From>> for Source { + fn from(value: Option>) -> Self { if let Some(v) = value { Self::Vec(v) } else { diff --git a/crates/brk_computer/src/grouped/type.rs b/crates/brk_computer/src/grouped/type.rs deleted file mode 100644 index df737cc71..000000000 --- a/crates/brk_computer/src/grouped/type.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::ops::{Add, AddAssign, Div}; - -use brk_vecs::StoredType; - -pub trait ComputedType -where - Self: - StoredType + From + Div + Add + AddAssign + Ord, -{ -} -impl ComputedType for T where - T: StoredType + From + Div + Add + AddAssign + Ord -{ -} diff --git a/crates/brk_computer/src/grouped/value_from_dateindex.rs b/crates/brk_computer/src/grouped/value_from_dateindex.rs index 08ba4ff50..88ab458c9 100644 --- a/crates/brk_computer/src/grouped/value_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/value_from_dateindex.rs @@ -1,11 +1,18 @@ use std::sync::Arc; -use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec}; +use brk_structs::{Bitcoin, DateIndex, Dollars, Sats, Version}; +use brk_vecs::{ + AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec, +}; -use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes}; +use crate::{ + Indexes, + grouped::ComputedVecsFromDateIndex, + indexes, price, + traits::{ComputeFromBitcoin, ComputeFromSats}, +}; use super::{Source, VecBuilderOptions}; @@ -30,7 +37,7 @@ impl ComputedValueVecsFromDateIndex { options: VecBuilderOptions, compute_dollars: bool, indexes: &indexes::Vecs, - ) -> color_eyre::Result { + ) -> Result { Ok(Self { sats: ComputedVecsFromDateIndex::forced_import( file, @@ -72,11 +79,11 @@ impl ComputedValueVecsFromDateIndex { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, mut compute: F, - ) -> color_eyre::Result<()> + ) -> Result<()> where F: FnMut( &mut EagerVec, @@ -95,7 +102,7 @@ impl ComputedValueVecsFromDateIndex { )?; let dateindex: Option<&StoredVec> = None; - self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, dateindex)?; + self.compute_rest(indexer, indexes, price, starting_indexes, exit, dateindex)?; Ok(()) } @@ -104,11 +111,11 @@ impl ComputedValueVecsFromDateIndex { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, dateindex: Option<&impl CollectableVec>, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if let Some(dateindex) = dateindex { self.sats .compute_rest(indexes, starting_indexes, exit, Some(dateindex))?; @@ -144,7 +151,7 @@ impl ComputedValueVecsFromDateIndex { } let dateindex_to_bitcoin = self.bitcoin.dateindex.as_ref().unwrap(); - let dateindex_to_close = fetched + let dateindex_to_close = price .as_ref() .unwrap() .timeindexes_to_close diff --git a/crates/brk_computer/src/grouped/value_from_height.rs b/crates/brk_computer/src/grouped/value_from_height.rs index 3eb6b92a2..7fbd8a47b 100644 --- a/crates/brk_computer/src/grouped/value_from_height.rs +++ b/crates/brk_computer/src/grouped/value_from_height.rs @@ -1,11 +1,18 @@ use std::sync::Arc; -use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec}; +use brk_structs::{Bitcoin, Dollars, Height, Sats, Version}; +use brk_vecs::{ + AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec, +}; -use crate::{Indexes, fetched, grouped::Source, indexes}; +use crate::{ + Indexes, + grouped::Source, + indexes, price, + traits::{ComputeFromBitcoin, ComputeFromSats}, +}; use super::{ComputedVecsFromHeight, VecBuilderOptions}; @@ -30,7 +37,7 @@ impl ComputedValueVecsFromHeight { options: VecBuilderOptions, compute_dollars: bool, indexes: &indexes::Vecs, - ) -> color_eyre::Result { + ) -> Result { Ok(Self { sats: ComputedVecsFromHeight::forced_import( file, @@ -72,11 +79,11 @@ impl ComputedValueVecsFromHeight { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, mut compute: F, - ) -> color_eyre::Result<()> + ) -> Result<()> where F: FnMut( &mut EagerVec, @@ -95,7 +102,7 @@ impl ComputedValueVecsFromHeight { )?; let height: Option<&StoredVec> = None; - self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, height)?; + self.compute_rest(indexer, indexes, price, starting_indexes, exit, height)?; Ok(()) } @@ -104,11 +111,11 @@ impl ComputedValueVecsFromHeight { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, height: Option<&impl CollectableVec>, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if let Some(height) = height { self.sats .compute_rest(indexes, starting_indexes, exit, Some(height))?; @@ -144,7 +151,7 @@ impl ComputedValueVecsFromHeight { } let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap(); - let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height; + let height_to_close = &price.as_ref().unwrap().chainindexes_to_close.height; if let Some(dollars) = self.dollars.as_mut() { dollars.compute_all( diff --git a/crates/brk_computer/src/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs index a651506c8..1a28f8763 100644 --- a/crates/brk_computer/src/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -1,14 +1,14 @@ use std::sync::Arc; -use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; +use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; use brk_vecs::{ - AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3, - File, Format, LazyVecFrom1, StoredIndex, StoredVec, + AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Computation, ComputedVecFrom3, + Exit, File, Format, LazyVecFrom1, StoredIndex, StoredVec, }; -use crate::{Indexes, fetched, grouped::Source, indexes}; +use crate::{Indexes, grouped::Source, indexes, price}; use super::{ComputedVecsFromTxindex, VecBuilderOptions}; @@ -45,10 +45,10 @@ impl ComputedValueVecsFromTxindex { version: Version, computation: Computation, format: Format, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, options: VecBuilderOptions, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); + ) -> Result { + let compute_dollars = price.is_some(); let name_in_btc = format!("{name}_in_btc"); let name_in_usd = format!("{name}_in_usd"); @@ -89,7 +89,7 @@ impl ComputedValueVecsFromTxindex { options, )?; - let dollars_txindex = fetched.map(|fetched| { + let dollars_txindex = price.map(|price| { ComputedVecFrom3::forced_import_or_init_from_3( computation, file, @@ -98,7 +98,7 @@ impl ComputedValueVecsFromTxindex { format, bitcoin_txindex.boxed_clone(), indexes.txindex_to_height.boxed_clone(), - fetched.chainindexes_to_close.height.boxed_clone(), + price.chainindexes_to_close.height.boxed_clone(), |txindex: TxIndex, txindex_to_btc_iter, txindex_to_height_iter, @@ -145,11 +145,11 @@ impl ComputedValueVecsFromTxindex { // &mut self, // indexer: &Indexer, // indexes: &indexes::Vecs, - // fetched: Option<&marketprice::Vecs>, + // price: Option<&marketprice::Vecs>, // starting_indexes: &Indexes, // exit: &Exit, // mut compute: F, - // ) -> color_eyre::Result<()> + // ) -> Result<()> // where // F: FnMut( // &mut EagerVec, @@ -187,8 +187,8 @@ impl ComputedValueVecsFromTxindex { starting_indexes: &Indexes, exit: &Exit, txindex: Option<&impl CollectableVec>, - fetched: Option<&fetched::Vecs>, - ) -> color_eyre::Result<()> { + price: Option<&price::Vecs>, + ) -> Result<()> { if let Some(txindex) = txindex { self.sats .compute_rest(indexer, indexes, starting_indexes, exit, Some(txindex))?; @@ -223,7 +223,7 @@ impl ComputedValueVecsFromTxindex { exit, &self.bitcoin, Some(dollars_txindex), - fetched.as_ref().unwrap(), + price.as_ref().unwrap(), )?; } diff --git a/crates/brk_computer/src/grouped/value_height.rs b/crates/brk_computer/src/grouped/value_height.rs index 77c7e6378..4fc0f920b 100644 --- a/crates/brk_computer/src/grouped/value_height.rs +++ b/crates/brk_computer/src/grouped/value_height.rs @@ -1,11 +1,16 @@ use std::sync::Arc; -use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, File, Format, StoredVec}; +use brk_structs::{Bitcoin, Dollars, Height, Sats, Version}; +use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, Exit, File, Format, StoredVec}; -use crate::{Indexes, fetched, grouped::Source, indexes}; +use crate::{ + Indexes, + grouped::Source, + indexes, price, + traits::{ComputeFromBitcoin, ComputeFromSats}, +}; #[derive(Clone)] pub struct ComputedHeightValueVecs { @@ -24,7 +29,7 @@ impl ComputedHeightValueVecs { version: Version, format: Format, compute_dollars: bool, - ) -> color_eyre::Result { + ) -> Result { Ok(Self { sats: source.is_compute().then(|| { EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format) @@ -52,11 +57,11 @@ impl ComputedHeightValueVecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, mut compute: F, - ) -> color_eyre::Result<()> + ) -> Result<()> where F: FnMut( &mut EagerVec, @@ -75,18 +80,18 @@ impl ComputedHeightValueVecs { )?; let height: Option<&StoredVec> = None; - self.compute_rest(fetched, starting_indexes, exit, height)?; + self.compute_rest(price, starting_indexes, exit, height)?; Ok(()) } pub fn compute_rest( &mut self, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, height: Option<&impl CollectableVec>, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if let Some(height) = height { self.bitcoin .compute_from_sats(starting_indexes.height, height, exit)?; @@ -99,7 +104,7 @@ impl ComputedHeightValueVecs { } let height_to_bitcoin = &self.bitcoin; - let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height; + let height_to_close = &price.as_ref().unwrap().chainindexes_to_close.height; if let Some(dollars) = self.dollars.as_mut() { dollars.compute_from_bitcoin( diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index 853905d2c..6b851f1c1 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -1,36 +1,38 @@ -use std::{ops::Deref, sync::Arc}; +use std::{ops::Deref, path::Path, sync::Arc}; -use brk_core::{ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_structs::{ Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height, InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2AAddressIndex, P2ABytes, P2MSOutputIndex, P2PK33AddressIndex, P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex, P2PKHBytes, P2SHAddressIndex, P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, - P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredUsize, + P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64, Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex, }; -use brk_exit::Exit; -use brk_indexer::Indexer; use brk_vecs::{ - AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1, - ComputedVecFrom2, EagerVec, File, Format, StoredIndex, VecIterator, + AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1, + ComputedVecFrom2, EagerVec, Exit, File, Format, StoredIndex, VecIterator, }; const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { + file: Arc, + pub dateindex_to_date: EagerVec, pub dateindex_to_dateindex: EagerVec, pub dateindex_to_first_height: EagerVec, - pub dateindex_to_height_count: EagerVec, + pub dateindex_to_height_count: EagerVec, pub dateindex_to_monthindex: EagerVec, pub dateindex_to_weekindex: EagerVec, pub decadeindex_to_decadeindex: EagerVec, pub decadeindex_to_first_yearindex: EagerVec, - pub decadeindex_to_yearindex_count: EagerVec, + pub decadeindex_to_yearindex_count: EagerVec, pub difficultyepoch_to_difficultyepoch: EagerVec, pub difficultyepoch_to_first_height: EagerVec, - pub difficultyepoch_to_height_count: EagerVec, + pub difficultyepoch_to_height_count: EagerVec, pub emptyoutputindex_to_emptyoutputindex: ComputedVecFrom1, pub halvingepoch_to_first_height: EagerVec, @@ -42,9 +44,9 @@ pub struct Vecs { pub height_to_halvingepoch: EagerVec, pub height_to_height: EagerVec, pub height_to_timestamp_fixed: EagerVec, - pub height_to_txindex_count: EagerVec, + pub height_to_txindex_count: EagerVec, pub inputindex_to_inputindex: ComputedVecFrom1, - pub monthindex_to_dateindex_count: EagerVec, + pub monthindex_to_dateindex_count: EagerVec, pub monthindex_to_first_dateindex: EagerVec, pub monthindex_to_monthindex: EagerVec, pub monthindex_to_quarterindex: EagerVec, @@ -73,39 +75,41 @@ pub struct Vecs { pub p2wshaddressindex_to_p2wshaddressindex: ComputedVecFrom1, pub quarterindex_to_first_monthindex: EagerVec, - pub quarterindex_to_monthindex_count: EagerVec, + pub quarterindex_to_monthindex_count: EagerVec, pub quarterindex_to_quarterindex: EagerVec, pub semesterindex_to_first_monthindex: EagerVec, - pub semesterindex_to_monthindex_count: EagerVec, + pub semesterindex_to_monthindex_count: EagerVec, pub semesterindex_to_semesterindex: EagerVec, pub txindex_to_height: EagerVec, pub txindex_to_input_count: - ComputedVecFrom2, + ComputedVecFrom2, pub txindex_to_output_count: - ComputedVecFrom2, + ComputedVecFrom2, pub txindex_to_txindex: ComputedVecFrom1, pub unknownoutputindex_to_unknownoutputindex: ComputedVecFrom1, - pub weekindex_to_dateindex_count: EagerVec, + pub weekindex_to_dateindex_count: EagerVec, pub weekindex_to_first_dateindex: EagerVec, pub weekindex_to_weekindex: EagerVec, pub yearindex_to_decadeindex: EagerVec, pub yearindex_to_first_monthindex: EagerVec, - pub yearindex_to_monthindex_count: EagerVec, + pub yearindex_to_monthindex_count: EagerVec, pub yearindex_to_yearindex: EagerVec, } impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, indexer: &Indexer, computation: Computation, format: Format, - ) -> color_eyre::Result { + ) -> Result { + let file = Arc::new(File::open(&parent.join("indexes"))?); + let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "outputindex", version + VERSION + Version::ZERO, format, @@ -115,7 +119,7 @@ impl Vecs { let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "inputindex", version + VERSION + Version::ZERO, format, @@ -125,7 +129,7 @@ impl Vecs { let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "txindex", version + VERSION + Version::ZERO, format, @@ -135,7 +139,7 @@ impl Vecs { let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2( computation, - file, + &file, "input_count", version + VERSION + Version::ZERO, format, @@ -151,14 +155,14 @@ impl Vecs { .next_at(txindex + 1) .map(|(_, v)| usize::from(v.into_owned())) .unwrap_or_else(|| inputindex_to_outputindex_iter.len()); - StoredUsize::from((start..end).count()) + StoredU64::from((start..end).count()) }) }, )?; let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2( computation, - file, + &file, "output_count", version + VERSION + Version::ZERO, format, @@ -174,14 +178,14 @@ impl Vecs { .next_at(txindex + 1) .map(|(_, v)| usize::from(v.into_owned())) .unwrap_or_else(|| outputindex_to_value_iter.len()); - StoredUsize::from((start..end).count()) + StoredU64::from((start..end).count()) }) }, )?; let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2pk33addressindex", version + VERSION + Version::ZERO, format, @@ -190,7 +194,7 @@ impl Vecs { )?; let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2pk65addressindex", version + VERSION + Version::ZERO, format, @@ -199,7 +203,7 @@ impl Vecs { )?; let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2pkhaddressindex", version + VERSION + Version::ZERO, format, @@ -208,7 +212,7 @@ impl Vecs { )?; let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2shaddressindex", version + VERSION + Version::ZERO, format, @@ -217,7 +221,7 @@ impl Vecs { )?; let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2traddressindex", version + VERSION + Version::ZERO, format, @@ -226,7 +230,7 @@ impl Vecs { )?; let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2wpkhaddressindex", version + VERSION + Version::ZERO, format, @@ -235,7 +239,7 @@ impl Vecs { )?; let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2wshaddressindex", version + VERSION + Version::ZERO, format, @@ -244,7 +248,7 @@ impl Vecs { )?; let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2aaddressindex", version + VERSION + Version::ZERO, format, @@ -253,7 +257,7 @@ impl Vecs { )?; let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "p2msoutputindex", version + VERSION + Version::ZERO, format, @@ -262,7 +266,7 @@ impl Vecs { )?; let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "emptyoutputindex", version + VERSION + Version::ZERO, format, @@ -271,7 +275,7 @@ impl Vecs { )?; let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "unknownoutputindex", version + VERSION + Version::ZERO, format, @@ -280,7 +284,7 @@ impl Vecs { )?; let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "opreturnindex", version + VERSION + Version::ZERO, format, @@ -308,263 +312,265 @@ impl Vecs { unknownoutputindex_to_unknownoutputindex, dateindex_to_date: EagerVec::forced_import( - file, + &file, "date", version + VERSION + Version::ZERO, format, )?, dateindex_to_dateindex: EagerVec::forced_import( - file, + &file, "dateindex", version + VERSION + Version::ZERO, format, )?, dateindex_to_first_height: EagerVec::forced_import( - file, + &file, "first_height", version + VERSION + Version::ZERO, format, )?, dateindex_to_monthindex: EagerVec::forced_import( - file, + &file, "monthindex", version + VERSION + Version::ZERO, format, )?, dateindex_to_weekindex: EagerVec::forced_import( - file, + &file, "weekindex", version + VERSION + Version::ZERO, format, )?, decadeindex_to_decadeindex: EagerVec::forced_import( - file, + &file, "decadeindex", version + VERSION + Version::ZERO, format, )?, decadeindex_to_first_yearindex: EagerVec::forced_import( - file, + &file, "first_yearindex", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_difficultyepoch: EagerVec::forced_import( - file, + &file, "difficultyepoch", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_first_height: EagerVec::forced_import( - file, + &file, "first_height", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_first_height: EagerVec::forced_import( - file, + &file, "first_height", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_halvingepoch: EagerVec::forced_import( - file, + &file, "halvingepoch", version + VERSION + Version::ZERO, format, )?, height_to_date: EagerVec::forced_import( - file, + &file, "date", version + VERSION + Version::ZERO, format, )?, height_to_difficultyepoch: EagerVec::forced_import( - file, + &file, "difficultyepoch", version + VERSION + Version::ZERO, format, )?, height_to_halvingepoch: EagerVec::forced_import( - file, + &file, "halvingepoch", version + VERSION + Version::ZERO, format, )?, height_to_height: EagerVec::forced_import( - file, + &file, "height", version + VERSION + Version::ZERO, format, )?, monthindex_to_first_dateindex: EagerVec::forced_import( - file, + &file, "first_dateindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_monthindex: EagerVec::forced_import( - file, + &file, "monthindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_quarterindex: EagerVec::forced_import( - file, + &file, "quarterindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_semesterindex: EagerVec::forced_import( - file, + &file, "semesterindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_yearindex: EagerVec::forced_import( - file, + &file, "yearindex", version + VERSION + Version::ZERO, format, )?, quarterindex_to_first_monthindex: EagerVec::forced_import( - file, + &file, "first_monthindex", version + VERSION + Version::ZERO, format, )?, semesterindex_to_first_monthindex: EagerVec::forced_import( - file, + &file, "first_monthindex", version + VERSION + Version::ZERO, format, )?, weekindex_to_first_dateindex: EagerVec::forced_import( - file, + &file, "first_dateindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_first_monthindex: EagerVec::forced_import( - file, + &file, "first_monthindex", version + VERSION + Version::ZERO, format, )?, quarterindex_to_quarterindex: EagerVec::forced_import( - file, + &file, "quarterindex", version + VERSION + Version::ZERO, format, )?, semesterindex_to_semesterindex: EagerVec::forced_import( - file, + &file, "semesterindex", version + VERSION + Version::ZERO, format, )?, weekindex_to_weekindex: EagerVec::forced_import( - file, + &file, "weekindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_decadeindex: EagerVec::forced_import( - file, + &file, "decadeindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_yearindex: EagerVec::forced_import( - file, + &file, "yearindex", version + VERSION + Version::ZERO, format, )?, height_to_date_fixed: EagerVec::forced_import( - file, + &file, "date_fixed", version + VERSION + Version::ZERO, format, )?, height_to_dateindex: EagerVec::forced_import( - file, + &file, "dateindex", version + VERSION + Version::ZERO, format, )?, txindex_to_height: EagerVec::forced_import( - file, + &file, "height", version + VERSION + Version::ZERO, format, )?, height_to_timestamp_fixed: EagerVec::forced_import( - file, + &file, "timestamp_fixed", version + VERSION + Version::ZERO, format, )?, height_to_txindex_count: EagerVec::forced_import( - file, + &file, "txindex_count", version + VERSION + Version::ZERO, format, )?, dateindex_to_height_count: EagerVec::forced_import( - file, + &file, "height_count", version + VERSION + Version::ZERO, format, )?, weekindex_to_dateindex_count: EagerVec::forced_import( - file, + &file, "dateindex_count", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_height_count: EagerVec::forced_import( - file, + &file, "height_count", version + VERSION + Version::ZERO, format, )?, monthindex_to_dateindex_count: EagerVec::forced_import( - file, + &file, "dateindex_count", version + VERSION + Version::ZERO, format, )?, quarterindex_to_monthindex_count: EagerVec::forced_import( - file, + &file, "monthindex_count", version + VERSION + Version::ZERO, format, )?, semesterindex_to_monthindex_count: EagerVec::forced_import( - file, + &file, "monthindex_count", version + VERSION + Version::ZERO, format, )?, yearindex_to_monthindex_count: EagerVec::forced_import( - file, + &file, "monthindex_count", version + VERSION + Version::ZERO, format, )?, decadeindex_to_yearindex_count: EagerVec::forced_import( - file, + &file, "yearindex_count", version + VERSION + Version::ZERO, format, )?, outputindex_to_txindex: EagerVec::forced_import( - file, + &file, "txindex", version + VERSION + Version::ZERO, format, )?, + + file, }) } @@ -573,7 +579,7 @@ impl Vecs { indexer: &Indexer, starting_indexes: brk_indexer::Indexes, exit: &Exit, - ) -> color_eyre::Result { + ) -> Result { // --- // OutputIndex // --- @@ -1110,6 +1116,9 @@ impl Vecs { exit, )?; + self.file.flush()?; + self.file.punch_holes()?; + Ok(Indexes { indexes: starting_indexes, dateindex: starting_dateindex, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 1363c8785..28388ece8 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -3,16 +3,15 @@ #![doc = include_str!("../examples/main.rs")] #![doc = "```"] -use std::{path::Path, sync::Arc}; +use std::path::Path; -use brk_core::Version; -use brk_exit::Exit; +use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vecs::{Computation, File, Format, PAGE_SIZE}; +use brk_structs::Version; +use brk_vecs::{AnyCollectableVec, Computation, Exit, Format}; use log::info; -mod all; mod blocks; mod cointime; mod constants; @@ -21,8 +20,10 @@ mod grouped; mod indexes; mod market; mod mining; +mod price; mod stateful; mod states; +mod traits; mod transactions; mod utils; @@ -32,9 +33,16 @@ use states::*; #[derive(Clone)] pub struct Computer { - file: Arc, - fetcher: Option, - pub vecs: all::Vecs, + pub indexes: indexes::Vecs, + pub constants: constants::Vecs, + pub blocks: blocks::Vecs, + pub mining: mining::Vecs, + pub market: market::Vecs, + pub price: Option, + pub transactions: transactions::Vecs, + pub stateful: stateful::Vecs, + pub fetched: Option, + pub cointime: cointime::Vecs, } const VERSION: Version = Version::ONE; @@ -42,50 +50,193 @@ const VERSION: Version = Version::ONE; impl Computer { /// Do NOT import multiple times or things will break !!! pub fn forced_import( - outputs_dir: &Path, + outputs_path: &Path, indexer: &Indexer, - computation: Computation, fetcher: Option, - format: Format, - ) -> color_eyre::Result { - let computed_path = outputs_dir.join("computed"); - let states_path = computed_path.join("states"); + ) -> Result { + let computed_path = outputs_path.join("computed"); - let file = Arc::new(File::open(&computed_path.join("vecs"))?); - file.set_min_len(PAGE_SIZE * 100_000_000)?; - file.set_min_regions(50_000)?; + let computation = Computation::Lazy; + let format = Format::Compressed; - let file_fetched = Arc::new(File::open(&outputs_dir.join("fetched/vecs"))?); + let indexes = indexes::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + indexer, + computation, + format, + )?; - Ok(Self { - vecs: all::Vecs::import( - &file, + let fetched = fetcher.map(|fetcher| { + fetched::Vecs::forced_import(outputs_path, fetcher, VERSION + Version::ZERO).unwrap() + }); + + let price = fetched.is_some().then(|| { + price::Vecs::forced_import( + &computed_path, VERSION + Version::ZERO, - indexer, - fetcher.is_some(), computation, format, - &file_fetched, - &states_path, + &indexes, + ) + .unwrap() + }); + + Ok(Self { + blocks: blocks::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + computation, + format, + &indexes, )?, - fetcher, - file, + mining: mining::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + computation, + format, + &indexes, + )?, + constants: constants::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + computation, + format, + &indexes, + )?, + market: market::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + computation, + format, + &indexes, + )?, + stateful: stateful::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + computation, + format, + &indexes, + price.as_ref(), + &computed_path.join("states"), + )?, + transactions: transactions::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + indexer, + &indexes, + computation, + format, + price.as_ref(), + )?, + cointime: cointime::Vecs::forced_import( + &computed_path, + VERSION + Version::ZERO, + computation, + format, + &indexes, + price.as_ref(), + )?, + indexes, + fetched, + price, }) } -} -impl Computer { pub fn compute( &mut self, - indexer: &mut Indexer, + indexer: &Indexer, starting_indexes: brk_indexer::Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { - info!("Computing..."); - self.vecs - .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)?; - self.file.flush()?; - self.file.punch_holes()?; + ) -> Result<()> { + info!("Computing indexes..."); + let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; + + info!("Computing constants..."); + self.constants + .compute(indexer, &self.indexes, &starting_indexes, exit)?; + + info!("Computing blocks..."); + self.blocks + .compute(indexer, &self.indexes, &starting_indexes, exit)?; + + info!("Computing mining..."); + self.mining + .compute(indexer, &self.indexes, &starting_indexes, exit)?; + + if let Some(fetched) = self.fetched.as_mut() { + info!("Computing fetched..."); + fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?; + + self.price.as_mut().unwrap().compute( + indexer, + &self.indexes, + &starting_indexes, + fetched, + exit, + )?; + } + + info!("Computing transactions..."); + self.transactions.compute( + indexer, + &self.indexes, + &starting_indexes, + self.price.as_ref(), + exit, + )?; + + if let Some(price) = self.price.as_ref() { + info!("Computing market..."); + self.market.compute( + indexer, + &self.indexes, + price, + &mut self.transactions, + &starting_indexes, + exit, + )?; + } + + info!("Computing stateful..."); + self.stateful.compute( + indexer, + &self.indexes, + &self.transactions, + self.price.as_ref(), + &self.market, + &mut starting_indexes, + exit, + )?; + + self.cointime.compute( + indexer, + &self.indexes, + &starting_indexes, + self.price.as_ref(), + &self.transactions, + &self.stateful, + exit, + )?; + Ok(()) } + + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + [ + self.constants.vecs(), + self.indexes.vecs(), + self.blocks.vecs(), + self.mining.vecs(), + self.market.vecs(), + self.transactions.vecs(), + self.stateful.vecs(), + self.cointime.vecs(), + self.fetched.as_ref().map_or(vec![], |v| v.vecs()), + self.price.as_ref().map_or(vec![], |v| v.vecs()), + ] + .into_iter() + .flatten() + .collect::>() + } } diff --git a/crates/brk_computer/src/market.rs b/crates/brk_computer/src/market.rs index a96e1435a..dfe85f6c4 100644 --- a/crates/brk_computer/src/market.rs +++ b/crates/brk_computer/src/market.rs @@ -1,14 +1,20 @@ -use std::{sync::Arc, thread}; +use std::{path::Path, sync::Arc, thread}; -use brk_core::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredUsize, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, Computation, EagerVec, File, Format, StoredIndex, VecIterator}; +use brk_structs::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version}; +use brk_vecs::{ + AnyCollectableVec, Computation, EagerVec, Exit, File, Format, StoredIndex, VecIterator, +}; -use crate::grouped::Source; +use crate::{ + grouped::Source, + price, + traits::{ComputeDCAAveragePriceViaLen, ComputeDCAStackViaLen, ComputeDrawdown}, +}; use super::{ - Indexes, fetched, + Indexes, grouped::{ComputedRatioVecsFromDateIndex, ComputedVecsFromDateIndex, VecBuilderOptions}, indexes, transactions, }; @@ -17,14 +23,16 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { + file: Arc, + pub height_to_marketcap: EagerVec, pub height_to_ath: EagerVec, pub height_to_drawdown: EagerVec, pub indexes_to_marketcap: ComputedVecsFromDateIndex, pub indexes_to_ath: ComputedVecsFromDateIndex, pub indexes_to_drawdown: ComputedVecsFromDateIndex, - pub indexes_to_days_since_ath: ComputedVecsFromDateIndex, - pub indexes_to_max_days_between_aths: ComputedVecsFromDateIndex, + pub indexes_to_days_since_ath: ComputedVecsFromDateIndex, + pub indexes_to_max_days_between_aths: ComputedVecsFromDateIndex, pub indexes_to_max_years_between_aths: ComputedVecsFromDateIndex, pub indexes_to_1w_sma: ComputedRatioVecsFromDateIndex, @@ -163,33 +171,35 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, - ) -> color_eyre::Result { + ) -> Result { + let file = Arc::new(File::open(&parent.join("market"))?); + Ok(Self { height_to_marketcap: EagerVec::forced_import( - file, + &file, "marketcap", version + VERSION + Version::ZERO, format, )?, height_to_ath: EagerVec::forced_import( - file, + &file, "ath", version + VERSION + Version::ZERO, format, )?, height_to_drawdown: EagerVec::forced_import( - file, + &file, "drawdown", version + VERSION + Version::ZERO, format, )?, indexes_to_marketcap: ComputedVecsFromDateIndex::forced_import( - file, + &file, "marketcap", Source::Compute, version + VERSION + Version::ZERO, @@ -199,7 +209,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_ath: ComputedVecsFromDateIndex::forced_import( - file, + &file, "ath", Source::Compute, version + VERSION + Version::ZERO, @@ -209,7 +219,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_drawdown: ComputedVecsFromDateIndex::forced_import( - file, + &file, "drawdown", Source::Compute, version + VERSION + Version::ZERO, @@ -219,7 +229,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_days_since_ath: ComputedVecsFromDateIndex::forced_import( - file, + &file, "days_since_ath", Source::Compute, version + VERSION + Version::ZERO, @@ -229,7 +239,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_max_days_between_aths: ComputedVecsFromDateIndex::forced_import( - file, + &file, "max_days_between_aths", Source::Compute, version + VERSION + Version::ZERO, @@ -239,7 +249,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_max_years_between_aths: ComputedVecsFromDateIndex::forced_import( - file, + &file, "max_years_between_aths", Source::Compute, version + VERSION + Version::ZERO, @@ -250,7 +260,7 @@ impl Vecs { )?, indexes_to_1w_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "1w_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -260,7 +270,7 @@ impl Vecs { true, )?, indexes_to_8d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "8d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -270,7 +280,7 @@ impl Vecs { true, )?, indexes_to_13d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "13d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -280,7 +290,7 @@ impl Vecs { true, )?, indexes_to_21d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "21d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -290,7 +300,7 @@ impl Vecs { true, )?, indexes_to_1m_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "1m_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -300,7 +310,7 @@ impl Vecs { true, )?, indexes_to_34d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "34d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -310,7 +320,7 @@ impl Vecs { true, )?, indexes_to_55d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "55d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -320,7 +330,7 @@ impl Vecs { true, )?, indexes_to_89d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "89d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -330,7 +340,7 @@ impl Vecs { true, )?, indexes_to_144d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "144d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -340,7 +350,7 @@ impl Vecs { true, )?, indexes_to_200d_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "200d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -350,7 +360,7 @@ impl Vecs { true, )?, indexes_to_1y_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "1y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -360,7 +370,7 @@ impl Vecs { true, )?, indexes_to_2y_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "2y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -370,7 +380,7 @@ impl Vecs { true, )?, indexes_to_200w_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "200w_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -380,7 +390,7 @@ impl Vecs { true, )?, indexes_to_4y_sma: ComputedRatioVecsFromDateIndex::forced_import( - file, + &file, "4y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -391,7 +401,7 @@ impl Vecs { )?, _1d_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1d_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -401,7 +411,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1w_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -411,7 +421,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -421,7 +431,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -431,7 +441,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -441,7 +451,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -451,7 +461,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "2y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -461,7 +471,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -471,7 +481,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "4y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -481,7 +491,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "5y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -491,7 +501,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -501,7 +511,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "8y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -511,7 +521,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "10y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -521,7 +531,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "2y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -531,7 +541,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -541,7 +551,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "4y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -551,7 +561,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "5y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -561,7 +571,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -571,7 +581,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "8y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -581,7 +591,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "10y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -592,7 +602,7 @@ impl Vecs { )?, _1w_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1w_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -602,7 +612,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -612,7 +622,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -622,7 +632,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -632,7 +642,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -642,7 +652,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "2y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -652,7 +662,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -662,7 +672,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "4y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -672,7 +682,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "5y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -682,7 +692,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -692,7 +702,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "8y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -702,7 +712,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "10y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -712,7 +722,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "2y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -722,7 +732,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -732,7 +742,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "4y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -742,7 +752,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "5y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -752,7 +762,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -762,7 +772,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "8y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -772,7 +782,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - file, + &file, "10y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -782,7 +792,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1w_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -792,7 +802,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -802,7 +812,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -812,7 +822,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -822,7 +832,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -832,7 +842,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "2y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -842,7 +852,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -852,7 +862,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "4y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -862,7 +872,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "5y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -872,7 +882,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -882,7 +892,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "8y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -892,7 +902,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "10y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -902,7 +912,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1d_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_1d_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -912,7 +922,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1w_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_1w_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -922,7 +932,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1m_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_1m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -932,7 +942,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_3m_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_3m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -942,7 +952,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_6m_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_6m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -952,7 +962,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_1y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -962,7 +972,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_2y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_2y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -972,7 +982,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_3y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_3y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -982,7 +992,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_4y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_4y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -992,7 +1002,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_5y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_5y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1002,7 +1012,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_6y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_6y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1012,7 +1022,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_8y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_8y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1022,7 +1032,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_10y_ago: ComputedVecsFromDateIndex::forced_import( - file, + &file, "price_10y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1032,7 +1042,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1w_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1042,7 +1052,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1052,7 +1062,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1062,7 +1072,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1072,7 +1082,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "1y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1082,7 +1092,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "2y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1092,7 +1102,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "3y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1102,7 +1112,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "4y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1112,7 +1122,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "5y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1122,7 +1132,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "6y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1132,7 +1142,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "8y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1142,7 +1152,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "10y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1153,7 +1163,7 @@ impl Vecs { )?, dca_class_2025_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2025_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1163,7 +1173,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2024_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1173,7 +1183,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2023_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1183,7 +1193,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2022_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1193,7 +1203,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2021_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1203,7 +1213,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2020_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1213,7 +1223,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2019_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1223,7 +1233,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2018_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1233,7 +1243,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2017_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1243,7 +1253,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2016_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1253,7 +1263,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_stack: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2015_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1264,7 +1274,7 @@ impl Vecs { )?, dca_class_2025_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2025_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1274,7 +1284,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2024_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1284,7 +1294,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2023_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1294,7 +1304,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2022_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1304,7 +1314,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2021_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1314,7 +1324,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2020_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1324,7 +1334,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2019_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1334,7 +1344,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2018_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1344,7 +1354,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2017_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1354,7 +1364,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2016_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1364,7 +1374,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_avg_price: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2015_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1375,7 +1385,7 @@ impl Vecs { )?, dca_class_2025_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2025_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1385,7 +1395,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2024_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1395,7 +1405,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2023_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1405,7 +1415,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2022_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1415,7 +1425,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2021_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1425,7 +1435,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2020_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1435,7 +1445,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2019_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1445,7 +1455,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2018_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1455,7 +1465,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2017_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1465,7 +1475,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2016_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1475,7 +1485,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_returns: ComputedVecsFromDateIndex::forced_import( - file, + &file, "dca_class_2015_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1486,7 +1496,7 @@ impl Vecs { )?, indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex::forced_import( - file, + &file, "200d_sma_x2_4", Source::Compute, version + VERSION + Version::ZERO, @@ -1496,7 +1506,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex::forced_import( - file, + &file, "200d_sma_x0_8", Source::Compute, version + VERSION + Version::ZERO, @@ -1505,6 +1515,8 @@ impl Vecs { indexes, VecBuilderOptions::default().add_last(), )?, + + file, }) } @@ -1512,14 +1524,14 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: &fetched::Vecs, + price: &price::Vecs, transactions: &mut transactions::Vecs, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.height_to_marketcap.compute_multiply( starting_indexes.height, - &fetched.chainindexes_to_close.height, + &price.chainindexes_to_close.height, transactions .indexes_to_subsidy .bitcoin @@ -1529,12 +1541,12 @@ impl Vecs { )?; self.height_to_ath.compute_max( starting_indexes.height, - &fetched.chainindexes_to_high.height, + &price.chainindexes_to_high.height, exit, )?; self.height_to_drawdown.compute_drawdown( starting_indexes.height, - &fetched.chainindexes_to_close.height, + &price.chainindexes_to_close.height, &self.height_to_ath, exit, )?; @@ -1547,14 +1559,15 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_multiply( starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), + price.timeindexes_to_close.dateindex.as_ref().unwrap(), transactions .indexes_to_subsidy .bitcoin .dateindex .unwrap_cumulative(), exit, - ) + )?; + Ok(()) }, )?; @@ -1566,9 +1579,10 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_max( starting_indexes.dateindex, - fetched.timeindexes_to_high.dateindex.as_ref().unwrap(), + price.timeindexes_to_high.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -1580,10 +1594,11 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_drawdown( starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), + price.timeindexes_to_close.dateindex.as_ref().unwrap(), self.indexes_to_ath.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -1593,7 +1608,7 @@ impl Vecs { starting_indexes, exit, |v, _, _, starting_indexes, exit| { - let mut high_iter = fetched + let mut high_iter = price .timeindexes_to_high .dateindex .as_ref() @@ -1609,19 +1624,20 @@ impl Vecs { prev.replace(if i > 0 { slf.into_iter().unwrap_get_inner_(i - 1) } else { - StoredUsize::default() + StoredU16::default() }); } let days = if *high_iter.unwrap_get_inner(i) == ath { - StoredUsize::default() + StoredU16::default() } else { - prev.unwrap() + StoredUsize::from(1) + prev.unwrap() + StoredU16::new(1) }; prev.replace(days); (i, days) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1641,7 +1657,7 @@ impl Vecs { prev.replace(if i > 0 { slf.into_iter().unwrap_get_inner_(i - 1) } else { - StoredUsize::ZERO + StoredU16::ZERO }); } let max = prev.unwrap().max(days); @@ -1649,7 +1665,8 @@ impl Vecs { (i, max) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1667,7 +1684,8 @@ impl Vecs { .unwrap(), |(i, max, ..)| (i, StoredF32::from(*max as f64 / 365.0)), exit, - ) + )?; + Ok(()) }, )?; @@ -1722,7 +1740,7 @@ impl Vecs { ), ] .into_iter() - .try_for_each(|(days, ago, returns, cagr)| -> color_eyre::Result<()> { + .try_for_each(|(days, ago, returns, cagr)| -> Result<()> { ago.compute_all( indexer, indexes, @@ -1731,10 +1749,11 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_previous_value( starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), + price.timeindexes_to_close.dateindex.as_ref().unwrap(), days, exit, - ) + )?; + Ok(()) }, )?; @@ -1746,10 +1765,11 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_percentage_change( starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), + price.timeindexes_to_close.dateindex.as_ref().unwrap(), days, exit, - ) + )?; + Ok(()) }, )?; @@ -1765,7 +1785,8 @@ impl Vecs { returns.dateindex.as_ref().unwrap(), days, exit, - ) + )?; + Ok(()) }, )?; } @@ -1861,7 +1882,7 @@ impl Vecs { ] .into_iter() .try_for_each( - |(days, dca_stack, dca_avg_price, dca_returns, dca_cagr)| -> color_eyre::Result<()> { + |(days, dca_stack, dca_avg_price, dca_returns, dca_cagr)| -> Result<()> { dca_stack.compute_all( indexer, indexes, @@ -1870,10 +1891,11 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_dca_stack_via_len( starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), + price.timeindexes_to_close.dateindex.as_ref().unwrap(), days, exit, - ) + )?; + Ok(()) }, )?; @@ -1888,7 +1910,8 @@ impl Vecs { dca_stack.dateindex.as_ref().unwrap(), days, exit, - ) + )?; + Ok(()) }, )?; @@ -1900,10 +1923,11 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_percentage_difference( starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), + price.timeindexes_to_close.dateindex.as_ref().unwrap(), dca_avg_price.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -1919,7 +1943,8 @@ impl Vecs { dca_returns.dateindex.as_ref().unwrap(), days, exit, - ) + )?; + Ok(()) }, )?; } @@ -1997,60 +2022,61 @@ impl Vecs { ), ] .into_iter() - .try_for_each( - |(year, avg_price, returns, stack)| -> color_eyre::Result<()> { - let dateindex = DateIndex::try_from(Date::new(year, 1, 1)).unwrap(); + .try_for_each(|(year, avg_price, returns, stack)| -> Result<()> { + let dateindex = DateIndex::try_from(Date::new(year, 1, 1)).unwrap(); - stack.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_dca_stack_via_from( - starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), - dateindex, - exit, - ) - }, - )?; + stack.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_dca_stack_via_from( + starting_indexes.dateindex, + price.timeindexes_to_close.dateindex.as_ref().unwrap(), + dateindex, + exit, + )?; + Ok(()) + }, + )?; - avg_price.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_dca_avg_price_via_from( - starting_indexes.dateindex, - stack.dateindex.as_ref().unwrap(), - dateindex, - exit, - ) - }, - )?; + avg_price.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_dca_avg_price_via_from( + starting_indexes.dateindex, + stack.dateindex.as_ref().unwrap(), + dateindex, + exit, + )?; + Ok(()) + }, + )?; - returns.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - v.compute_percentage_difference( - starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), - avg_price.dateindex.as_ref().unwrap(), - exit, - ) - }, - )?; + returns.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_percentage_difference( + starting_indexes.dateindex, + price.timeindexes_to_close.dateindex.as_ref().unwrap(), + avg_price.dateindex.as_ref().unwrap(), + exit, + )?; + Ok(()) + }, + )?; - Ok(()) - }, - )?; + Ok(()) + })?; - thread::scope(|s| -> color_eyre::Result<()> { + thread::scope(|s| -> Result<()> { [ (&mut self.indexes_to_1w_sma, 7), (&mut self.indexes_to_8d_sma, 8), @@ -2069,20 +2095,21 @@ impl Vecs { ] .into_iter() .for_each(|(vecs, sma)| { - s.spawn(move || -> color_eyre::Result<()> { + s.spawn(move || -> Result<()> { vecs.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |v, _, _, starting_indexes, exit| { v.compute_sma( starting_indexes.dateindex, - fetched.timeindexes_to_close.dateindex.as_ref().unwrap(), + price.timeindexes_to_close.dateindex.as_ref().unwrap(), sma, exit, - ) + )?; + Ok(()) }, ) }); @@ -2107,7 +2134,8 @@ impl Vecs { .unwrap(), |(i, v, ..)| (i, v * 0.8), exit, - ) + )?; + Ok(()) }, )?; @@ -2128,10 +2156,13 @@ impl Vecs { .unwrap(), |(i, v, ..)| (i, v * 2.4), exit, - ) + )?; + Ok(()) }, )?; + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } diff --git a/crates/brk_computer/src/mining.rs b/crates/brk_computer/src/mining.rs index 52f7ddbfc..0e327d22d 100644 --- a/crates/brk_computer/src/mining.rs +++ b/crates/brk_computer/src/mining.rs @@ -1,9 +1,9 @@ -use std::sync::Arc; +use std::{path::Path, sync::Arc}; -use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator}; +use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version}; +use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator}; use crate::grouped::Source; @@ -17,6 +17,8 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { + file: Arc, + pub indexes_to_difficulty: ComputedVecsFromHeight, pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex, pub indexes_to_halvingepoch: ComputedVecsFromDateIndex, @@ -24,15 +26,17 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, - ) -> color_eyre::Result { + ) -> Result { + let file = Arc::new(File::open(&parent.join("mining"))?); + Ok(Self { indexes_to_difficulty: ComputedVecsFromHeight::forced_import( - file, + &file, "difficulty", Source::None, version + VERSION + Version::ZERO, @@ -42,7 +46,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import( - file, + &file, "difficultyepoch", Source::Compute, version + VERSION + Version::ZERO, @@ -52,7 +56,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import( - file, + &file, "halvingepoch", Source::Compute, version + VERSION + Version::ZERO, @@ -61,6 +65,8 @@ impl Vecs { indexes, VecBuilderOptions::default().add_last(), )?, + + file, }) } @@ -70,7 +76,7 @@ impl Vecs { indexes: &indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter(); self.indexes_to_difficultyepoch.compute_all( indexer, @@ -91,7 +97,8 @@ impl Vecs { ) }, exit, - ) + )?; + Ok(()) }, )?; @@ -115,7 +122,8 @@ impl Vecs { ) }, exit, - ) + )?; + Ok(()) }, )?; @@ -126,6 +134,8 @@ impl Vecs { Some(&indexer.vecs.height_to_difficulty), )?; + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } diff --git a/crates/brk_computer/src/price.rs b/crates/brk_computer/src/price.rs new file mode 100644 index 000000000..a5e338478 --- /dev/null +++ b/crates/brk_computer/src/price.rs @@ -0,0 +1,1273 @@ +use std::{path::Path, sync::Arc}; + +use brk_error::Result; +use brk_indexer::Indexer; +use brk_structs::{ + Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex, + OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex, +}; +use brk_vecs::{ + AnyCollectableVec, AnyIterableVec, AnyStoredVec, Computation, EagerVec, Exit, File, Format, + GenericStoredVec, RawVec, +}; + +use crate::{fetched, grouped::Source}; + +use super::{ + Indexes, + grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeightStrict, VecBuilderOptions}, + indexes, +}; + +#[derive(Clone)] +pub struct Vecs { + file: Arc, + + pub dateindex_to_close_in_cents: EagerVec>, + pub dateindex_to_high_in_cents: EagerVec>, + pub dateindex_to_low_in_cents: EagerVec>, + pub dateindex_to_ohlc: RawVec, + pub dateindex_to_ohlc_in_sats: RawVec, + pub dateindex_to_open_in_cents: EagerVec>, + pub height_to_close_in_cents: EagerVec>, + pub height_to_high_in_cents: EagerVec>, + pub height_to_low_in_cents: EagerVec>, + pub height_to_ohlc: RawVec, + pub height_to_ohlc_in_sats: RawVec, + pub height_to_open_in_cents: EagerVec>, + pub timeindexes_to_close: ComputedVecsFromDateIndex>, + pub timeindexes_to_high: ComputedVecsFromDateIndex>, + pub timeindexes_to_low: ComputedVecsFromDateIndex>, + pub timeindexes_to_open: ComputedVecsFromDateIndex>, + pub timeindexes_to_open_in_sats: ComputedVecsFromDateIndex>, + pub timeindexes_to_high_in_sats: ComputedVecsFromDateIndex>, + pub timeindexes_to_low_in_sats: ComputedVecsFromDateIndex>, + pub timeindexes_to_close_in_sats: ComputedVecsFromDateIndex>, + pub chainindexes_to_close: ComputedVecsFromHeightStrict>, + pub chainindexes_to_high: ComputedVecsFromHeightStrict>, + pub chainindexes_to_low: ComputedVecsFromHeightStrict>, + pub chainindexes_to_open: ComputedVecsFromHeightStrict>, + pub chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict>, + pub chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict>, + pub chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict>, + pub chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict>, + pub weekindex_to_ohlc: RawVec, + pub weekindex_to_ohlc_in_sats: RawVec, + pub difficultyepoch_to_ohlc: RawVec, + pub difficultyepoch_to_ohlc_in_sats: RawVec, + pub monthindex_to_ohlc: RawVec, + pub monthindex_to_ohlc_in_sats: RawVec, + pub quarterindex_to_ohlc: RawVec, + pub quarterindex_to_ohlc_in_sats: RawVec, + pub semesterindex_to_ohlc: RawVec, + pub semesterindex_to_ohlc_in_sats: RawVec, + pub yearindex_to_ohlc: RawVec, + pub yearindex_to_ohlc_in_sats: RawVec, + // pub halvingepoch_to_ohlc: StorableVec, + // pub halvingepoch_to_ohlc_in_sats: StorableVec, + pub decadeindex_to_ohlc: RawVec, + pub decadeindex_to_ohlc_in_sats: RawVec, +} + +const VERSION: Version = Version::ZERO; +const VERSION_IN_SATS: Version = Version::ZERO; + +impl Vecs { + pub fn forced_import( + parent: &Path, + version: Version, + computation: Computation, + format: Format, + indexes: &indexes::Vecs, + ) -> Result { + let file = Arc::new(File::open(&parent.join("price"))?); + + Ok(Self { + dateindex_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + dateindex_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + dateindex_to_close_in_cents: EagerVec::forced_import( + &file, + "close_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + dateindex_to_high_in_cents: EagerVec::forced_import( + &file, + "high_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + dateindex_to_low_in_cents: EagerVec::forced_import( + &file, + "low_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + dateindex_to_open_in_cents: EagerVec::forced_import( + &file, + "open_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + height_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + height_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + height_to_close_in_cents: EagerVec::forced_import( + &file, + "close_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + height_to_high_in_cents: EagerVec::forced_import( + &file, + "high_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + height_to_low_in_cents: EagerVec::forced_import( + &file, + "low_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + height_to_open_in_cents: EagerVec::forced_import( + &file, + "open_in_cents", + version + VERSION + Version::ZERO, + format, + )?, + timeindexes_to_open: ComputedVecsFromDateIndex::forced_import( + &file, + "open", + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_first(), + )?, + timeindexes_to_high: ComputedVecsFromDateIndex::forced_import( + &file, + "high", + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_max(), + )?, + timeindexes_to_low: ComputedVecsFromDateIndex::forced_import( + &file, + "low", + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_min(), + )?, + timeindexes_to_close: ComputedVecsFromDateIndex::forced_import( + &file, + "close", + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import( + &file, + "open_in_sats", + Source::Compute, + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_first(), + )?, + timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import( + &file, + "high_in_sats", + Source::Compute, + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_max(), + )?, + timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import( + &file, + "low_in_sats", + Source::Compute, + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_min(), + )?, + timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import( + &file, + "close_in_sats", + Source::Compute, + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( + &file, + "open", + version + VERSION + Version::ZERO, + format, + VecBuilderOptions::default().add_first(), + )?, + chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( + &file, + "high", + version + VERSION + Version::ZERO, + format, + VecBuilderOptions::default().add_max(), + )?, + chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( + &file, + "low", + version + VERSION + Version::ZERO, + format, + VecBuilderOptions::default().add_min(), + )?, + chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( + &file, + "close", + version + VERSION + Version::ZERO, + format, + VecBuilderOptions::default().add_last(), + )?, + chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import( + &file, + "open_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + VecBuilderOptions::default().add_first(), + )?, + chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import( + &file, + "high_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + VecBuilderOptions::default().add_max(), + )?, + chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import( + &file, + "low_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + VecBuilderOptions::default().add_min(), + )?, + chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import( + &file, + "close_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + VecBuilderOptions::default().add_last(), + )?, + weekindex_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + weekindex_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + difficultyepoch_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + difficultyepoch_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + monthindex_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + monthindex_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + quarterindex_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + quarterindex_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + semesterindex_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + semesterindex_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + yearindex_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + yearindex_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + // halvingepoch_to_ohlc: StorableVec::forced_import(file, + // "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?, + decadeindex_to_ohlc: RawVec::forced_import( + &file, + "ohlc", + version + VERSION + Version::ZERO, + )?, + decadeindex_to_ohlc_in_sats: RawVec::forced_import( + &file, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + )?, + + file, + }) + } + + pub fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + fetched: &fetched::Vecs, + exit: &Exit, + ) -> Result<()> { + self.height_to_open_in_cents.compute_transform( + starting_indexes.height, + &fetched.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + + self.height_to_high_in_cents.compute_transform( + starting_indexes.height, + &fetched.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + + self.height_to_low_in_cents.compute_transform( + starting_indexes.height, + &fetched.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + + self.height_to_close_in_cents.compute_transform( + starting_indexes.height, + &fetched.height_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + + fetched + .height_to_ohlc_in_cents + .iter_at(starting_indexes.height) + .try_for_each(|(i, v)| -> Result<()> { + self.height_to_ohlc + .forced_push_at(i, OHLCDollars::from(v.into_owned()), exit)?; + Ok(()) + })?; + self.height_to_ohlc.safe_flush(exit)?; + + self.dateindex_to_open_in_cents.compute_transform( + starting_indexes.dateindex, + &fetched.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + + self.dateindex_to_high_in_cents.compute_transform( + starting_indexes.dateindex, + &fetched.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + + self.dateindex_to_low_in_cents.compute_transform( + starting_indexes.dateindex, + &fetched.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + + self.dateindex_to_close_in_cents.compute_transform( + starting_indexes.dateindex, + &fetched.dateindex_to_ohlc_in_cents, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + + fetched + .dateindex_to_ohlc_in_cents + .iter_at(starting_indexes.dateindex) + .try_for_each(|(i, v)| -> Result<()> { + self.dateindex_to_ohlc.forced_push_at( + i, + OHLCDollars::from(v.into_owned()), + exit, + )?; + Ok(()) + })?; + self.dateindex_to_ohlc.safe_flush(exit)?; + + self.timeindexes_to_close.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + &self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + Ok(()) + }, + )?; + + self.timeindexes_to_high.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + &self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + Ok(()) + }, + )?; + + self.timeindexes_to_low.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + &self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + Ok(()) + }, + )?; + + self.timeindexes_to_open.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + &self.dateindex_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + Ok(()) + }, + )?; + + self.chainindexes_to_close.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.close), + exit, + )?; + Ok(()) + }, + )?; + + self.chainindexes_to_high.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.high), + exit, + )?; + Ok(()) + }, + )?; + + self.chainindexes_to_low.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.low), + exit, + )?; + Ok(()) + }, + )?; + + self.chainindexes_to_open.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.height_to_ohlc, + |(di, ohlc, ..)| (di, ohlc.open), + exit, + )?; + Ok(()) + }, + )?; + + let mut weekindex_first_iter = self.timeindexes_to_open.weekindex.unwrap_first().iter(); + let mut weekindex_max_iter = self.timeindexes_to_high.weekindex.unwrap_max().iter(); + let mut weekindex_min_iter = self.timeindexes_to_low.weekindex.unwrap_min().iter(); + self.timeindexes_to_close + .weekindex + .unwrap_last() + .iter_at(starting_indexes.weekindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + let open = weekindex_first_iter.unwrap_get_inner(i); + let high = weekindex_max_iter.unwrap_get_inner(i); + let low = weekindex_min_iter.unwrap_get_inner(i); + self.weekindex_to_ohlc.forced_push_at( + i, + OHLCDollars { + open, + high, + low, + close, + }, + exit, + )?; + Ok(()) + })?; + self.weekindex_to_ohlc.safe_flush(exit)?; + + let mut difficultyepoch_first_iter = self + .chainindexes_to_open + .difficultyepoch + .unwrap_first() + .iter(); + let mut difficultyepoch_max_iter = self + .chainindexes_to_high + .difficultyepoch + .unwrap_max() + .iter(); + let mut difficultyepoch_min_iter = + self.chainindexes_to_low.difficultyepoch.unwrap_min().iter(); + self.chainindexes_to_close + .difficultyepoch + .unwrap_last() + .iter_at(starting_indexes.difficultyepoch) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + let open = difficultyepoch_first_iter.unwrap_get_inner(i); + let high = difficultyepoch_max_iter.unwrap_get_inner(i); + let low = difficultyepoch_min_iter.unwrap_get_inner(i); + self.difficultyepoch_to_ohlc.forced_push_at( + i, + OHLCDollars { + open, + high, + low, + close, + }, + exit, + )?; + Ok(()) + })?; + self.difficultyepoch_to_ohlc.safe_flush(exit)?; + + let mut monthindex_first_iter = self.timeindexes_to_open.monthindex.unwrap_first().iter(); + let mut monthindex_max_iter = self.timeindexes_to_high.monthindex.unwrap_max().iter(); + let mut monthindex_min_iter = self.timeindexes_to_low.monthindex.unwrap_min().iter(); + self.timeindexes_to_close + .monthindex + .unwrap_last() + .iter_at(starting_indexes.monthindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + let open = monthindex_first_iter.unwrap_get_inner(i); + let high = monthindex_max_iter.unwrap_get_inner(i); + let low = monthindex_min_iter.unwrap_get_inner(i); + self.monthindex_to_ohlc.forced_push_at( + i, + OHLCDollars { + open, + high, + low, + close, + }, + exit, + )?; + Ok(()) + })?; + self.monthindex_to_ohlc.safe_flush(exit)?; + + let mut quarterindex_first_iter = + self.timeindexes_to_open.quarterindex.unwrap_first().iter(); + let mut quarterindex_max_iter = self.timeindexes_to_high.quarterindex.unwrap_max().iter(); + let mut quarterindex_min_iter = self.timeindexes_to_low.quarterindex.unwrap_min().iter(); + self.timeindexes_to_close + .quarterindex + .unwrap_last() + .iter_at(starting_indexes.quarterindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + let open = quarterindex_first_iter.unwrap_get_inner(i); + let high = quarterindex_max_iter.unwrap_get_inner(i); + let low = quarterindex_min_iter.unwrap_get_inner(i); + self.quarterindex_to_ohlc.forced_push_at( + i, + OHLCDollars { + open, + high, + low, + close, + }, + exit, + )?; + Ok(()) + })?; + self.quarterindex_to_ohlc.safe_flush(exit)?; + + let mut semesterindex_first_iter = + self.timeindexes_to_open.semesterindex.unwrap_first().iter(); + let mut semesterindex_max_iter = self.timeindexes_to_high.semesterindex.unwrap_max().iter(); + let mut semesterindex_min_iter = self.timeindexes_to_low.semesterindex.unwrap_min().iter(); + self.timeindexes_to_close + .semesterindex + .unwrap_last() + .iter_at(starting_indexes.semesterindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + let open = semesterindex_first_iter.unwrap_get_inner(i); + let high = semesterindex_max_iter.unwrap_get_inner(i); + let low = semesterindex_min_iter.unwrap_get_inner(i); + self.semesterindex_to_ohlc.forced_push_at( + i, + OHLCDollars { + open, + high, + low, + close, + }, + exit, + )?; + Ok(()) + })?; + self.semesterindex_to_ohlc.safe_flush(exit)?; + + let mut yearindex_first_iter = self.timeindexes_to_open.yearindex.unwrap_first().iter(); + let mut yearindex_max_iter = self.timeindexes_to_high.yearindex.unwrap_max().iter(); + let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter(); + self.timeindexes_to_close + .yearindex + .unwrap_last() + .iter_at(starting_indexes.yearindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + let open = yearindex_first_iter.unwrap_get_inner(i); + let high = yearindex_max_iter.unwrap_get_inner(i); + let low = yearindex_min_iter.unwrap_get_inner(i); + self.yearindex_to_ohlc.forced_push_at( + i, + OHLCDollars { + open, + high, + low, + close, + }, + exit, + )?; + Ok(()) + })?; + self.yearindex_to_ohlc.safe_flush(exit)?; + + // self.halvingepoch_to_ohlc + // .compute_transform(starting_indexes.halvingepoch, other, t, exit)?; + + let mut decadeindex_first_iter = self.timeindexes_to_open.decadeindex.unwrap_first().iter(); + let mut decadeindex_max_iter = self.timeindexes_to_high.decadeindex.unwrap_max().iter(); + let mut decadeindex_min_iter = self.timeindexes_to_low.decadeindex.unwrap_min().iter(); + self.timeindexes_to_close + .decadeindex + .unwrap_last() + .iter_at(starting_indexes.decadeindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + let open = decadeindex_first_iter.unwrap_get_inner(i); + let high = decadeindex_max_iter.unwrap_get_inner(i); + let low = decadeindex_min_iter.unwrap_get_inner(i); + self.decadeindex_to_ohlc.forced_push_at( + i, + OHLCDollars { + open, + high, + low, + close, + }, + exit, + )?; + Ok(()) + })?; + self.decadeindex_to_ohlc.safe_flush(exit)?; + + self.chainindexes_to_open_in_sats.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.chainindexes_to_open.height, + |(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)), + exit, + )?; + Ok(()) + }, + )?; + + self.chainindexes_to_high_in_sats.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.chainindexes_to_low.height, + |(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)), + exit, + )?; + Ok(()) + }, + )?; + + self.chainindexes_to_low_in_sats.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.chainindexes_to_high.height, + |(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)), + exit, + )?; + Ok(()) + }, + )?; + + self.chainindexes_to_close_in_sats.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.height, + &self.chainindexes_to_close.height, + |(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)), + exit, + )?; + Ok(()) + }, + )?; + + self.timeindexes_to_open_in_sats.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + self.timeindexes_to_open.dateindex.as_ref().unwrap(), + |(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)), + exit, + )?; + Ok(()) + }, + )?; + + self.timeindexes_to_high_in_sats.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + self.timeindexes_to_low.dateindex.as_ref().unwrap(), + |(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)), + exit, + )?; + Ok(()) + }, + )?; + + self.timeindexes_to_low_in_sats.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + self.timeindexes_to_high.dateindex.as_ref().unwrap(), + |(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)), + exit, + )?; + Ok(()) + }, + )?; + + self.timeindexes_to_close_in_sats.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_transform( + starting_indexes.dateindex, + self.timeindexes_to_close.dateindex.as_ref().unwrap(), + |(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)), + exit, + )?; + Ok(()) + }, + )?; + + let mut height_first_iter = self.chainindexes_to_open_in_sats.height.iter(); + let mut height_max_iter = self.chainindexes_to_high_in_sats.height.iter(); + let mut height_min_iter = self.chainindexes_to_low_in_sats.height.iter(); + self.chainindexes_to_close_in_sats + .height + .iter_at(starting_indexes.height) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.height_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: height_first_iter.unwrap_get_inner(i), + high: height_max_iter.unwrap_get_inner(i), + low: height_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.height_to_ohlc_in_sats.safe_flush(exit)?; + + let mut dateindex_first_iter = self + .timeindexes_to_open_in_sats + .dateindex + .as_ref() + .unwrap() + .iter(); + let mut dateindex_max_iter = self + .timeindexes_to_high_in_sats + .dateindex + .as_ref() + .unwrap() + .iter(); + let mut dateindex_min_iter = self + .timeindexes_to_low_in_sats + .dateindex + .as_ref() + .unwrap() + .iter(); + self.timeindexes_to_close_in_sats + .dateindex + .as_ref() + .unwrap() + .iter_at(starting_indexes.dateindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.dateindex_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: dateindex_first_iter.unwrap_get_inner(i), + high: dateindex_max_iter.unwrap_get_inner(i), + low: dateindex_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.dateindex_to_ohlc_in_sats.safe_flush(exit)?; + + let mut weekindex_first_iter = self + .timeindexes_to_open_in_sats + .weekindex + .unwrap_first() + .iter(); + let mut weekindex_max_iter = self + .timeindexes_to_high_in_sats + .weekindex + .unwrap_max() + .iter(); + let mut weekindex_min_iter = self + .timeindexes_to_low_in_sats + .weekindex + .unwrap_min() + .iter(); + self.timeindexes_to_close_in_sats + .weekindex + .unwrap_last() + .iter_at(starting_indexes.weekindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.weekindex_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: weekindex_first_iter.unwrap_get_inner(i), + high: weekindex_max_iter.unwrap_get_inner(i), + low: weekindex_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.weekindex_to_ohlc_in_sats.safe_flush(exit)?; + + let mut difficultyepoch_first_iter = self + .chainindexes_to_open_in_sats + .difficultyepoch + .unwrap_first() + .iter(); + let mut difficultyepoch_max_iter = self + .chainindexes_to_high_in_sats + .difficultyepoch + .unwrap_max() + .iter(); + let mut difficultyepoch_min_iter = self + .chainindexes_to_low_in_sats + .difficultyepoch + .unwrap_min() + .iter(); + self.chainindexes_to_close_in_sats + .difficultyepoch + .unwrap_last() + .iter_at(starting_indexes.difficultyepoch) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.difficultyepoch_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: difficultyepoch_first_iter.unwrap_get_inner(i), + high: difficultyepoch_max_iter.unwrap_get_inner(i), + low: difficultyepoch_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.difficultyepoch_to_ohlc_in_sats.safe_flush(exit)?; + + let mut monthindex_first_iter = self + .timeindexes_to_open_in_sats + .monthindex + .unwrap_first() + .iter(); + let mut monthindex_max_iter = self + .timeindexes_to_high_in_sats + .monthindex + .unwrap_max() + .iter(); + let mut monthindex_min_iter = self + .timeindexes_to_low_in_sats + .monthindex + .unwrap_min() + .iter(); + self.timeindexes_to_close_in_sats + .monthindex + .unwrap_last() + .iter_at(starting_indexes.monthindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.monthindex_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: monthindex_first_iter.unwrap_get_inner(i), + high: monthindex_max_iter.unwrap_get_inner(i), + low: monthindex_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.monthindex_to_ohlc_in_sats.safe_flush(exit)?; + + let mut quarterindex_first_iter = self + .timeindexes_to_open_in_sats + .quarterindex + .unwrap_first() + .iter(); + let mut quarterindex_max_iter = self + .timeindexes_to_high_in_sats + .quarterindex + .unwrap_max() + .iter(); + let mut quarterindex_min_iter = self + .timeindexes_to_low_in_sats + .quarterindex + .unwrap_min() + .iter(); + self.timeindexes_to_close_in_sats + .quarterindex + .unwrap_last() + .iter_at(starting_indexes.quarterindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.quarterindex_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: quarterindex_first_iter.unwrap_get_inner(i), + high: quarterindex_max_iter.unwrap_get_inner(i), + low: quarterindex_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.quarterindex_to_ohlc_in_sats.safe_flush(exit)?; + + let mut semesterindex_first_iter = self + .timeindexes_to_open_in_sats + .semesterindex + .unwrap_first() + .iter(); + let mut semesterindex_max_iter = self + .timeindexes_to_high_in_sats + .semesterindex + .unwrap_max() + .iter(); + let mut semesterindex_min_iter = self + .timeindexes_to_low_in_sats + .semesterindex + .unwrap_min() + .iter(); + self.timeindexes_to_close_in_sats + .semesterindex + .unwrap_last() + .iter_at(starting_indexes.semesterindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.semesterindex_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: semesterindex_first_iter.unwrap_get_inner(i), + high: semesterindex_max_iter.unwrap_get_inner(i), + low: semesterindex_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.semesterindex_to_ohlc_in_sats.safe_flush(exit)?; + + let mut yearindex_first_iter = self + .timeindexes_to_open_in_sats + .yearindex + .unwrap_first() + .iter(); + let mut yearindex_max_iter = self + .timeindexes_to_high_in_sats + .yearindex + .unwrap_max() + .iter(); + let mut yearindex_min_iter = self + .timeindexes_to_low_in_sats + .yearindex + .unwrap_min() + .iter(); + self.timeindexes_to_close_in_sats + .yearindex + .unwrap_last() + .iter_at(starting_indexes.yearindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.yearindex_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: yearindex_first_iter.unwrap_get_inner(i), + high: yearindex_max_iter.unwrap_get_inner(i), + low: yearindex_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.yearindex_to_ohlc_in_sats.safe_flush(exit)?; + + // self.halvingepoch_to_ohlc + // _in_sats.compute_transform(starting_indexes.halvingepoch, other, t, exit)?; + + let mut decadeindex_first_iter = self + .timeindexes_to_open_in_sats + .decadeindex + .unwrap_first() + .iter(); + let mut decadeindex_max_iter = self + .timeindexes_to_high_in_sats + .decadeindex + .unwrap_max() + .iter(); + let mut decadeindex_min_iter = self + .timeindexes_to_low_in_sats + .decadeindex + .unwrap_min() + .iter(); + self.timeindexes_to_close_in_sats + .decadeindex + .unwrap_last() + .iter_at(starting_indexes.decadeindex) + .try_for_each(|(i, v)| -> Result<()> { + let close = v.into_owned(); + self.decadeindex_to_ohlc_in_sats.forced_push_at( + i, + OHLCSats { + open: decadeindex_first_iter.unwrap_get_inner(i), + high: decadeindex_max_iter.unwrap_get_inner(i), + low: decadeindex_min_iter.unwrap_get_inner(i), + close, + }, + exit, + )?; + Ok(()) + })?; + self.decadeindex_to_ohlc_in_sats.safe_flush(exit)?; + + self.file.flush()?; + self.file.punch_holes()?; + Ok(()) + } + + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + vec![ + vec![ + &self.dateindex_to_close_in_cents as &dyn AnyCollectableVec, + &self.dateindex_to_high_in_cents, + &self.dateindex_to_low_in_cents, + &self.dateindex_to_ohlc, + &self.dateindex_to_open_in_cents, + &self.height_to_close_in_cents, + &self.height_to_high_in_cents, + &self.height_to_low_in_cents, + &self.height_to_ohlc, + &self.height_to_open_in_cents, + &self.weekindex_to_ohlc, + &self.difficultyepoch_to_ohlc, + &self.monthindex_to_ohlc, + &self.quarterindex_to_ohlc, + &self.semesterindex_to_ohlc, + &self.yearindex_to_ohlc, + // &self.halvingepoch_to_ohlc, + &self.decadeindex_to_ohlc, + &self.height_to_ohlc_in_sats, + &self.dateindex_to_ohlc_in_sats, + &self.weekindex_to_ohlc_in_sats, + &self.difficultyepoch_to_ohlc_in_sats, + &self.monthindex_to_ohlc_in_sats, + &self.quarterindex_to_ohlc_in_sats, + &self.semesterindex_to_ohlc_in_sats, + &self.yearindex_to_ohlc_in_sats, + // &self.halvingepoch_to_ohlc_in_sats, + &self.decadeindex_to_ohlc_in_sats, + ], + self.timeindexes_to_close.vecs(), + self.timeindexes_to_high.vecs(), + self.timeindexes_to_low.vecs(), + self.timeindexes_to_open.vecs(), + self.chainindexes_to_close.vecs(), + self.chainindexes_to_high.vecs(), + self.chainindexes_to_low.vecs(), + self.chainindexes_to_open.vecs(), + self.timeindexes_to_close_in_sats.vecs(), + self.timeindexes_to_high_in_sats.vecs(), + self.timeindexes_to_low_in_sats.vecs(), + self.timeindexes_to_open_in_sats.vecs(), + self.chainindexes_to_close_in_sats.vecs(), + self.chainindexes_to_high_in_sats.vecs(), + self.chainindexes_to_low_in_sats.vecs(), + self.chainindexes_to_open_in_sats.vecs(), + ] + .into_iter() + .flatten() + .collect::>() + } +} diff --git a/crates/brk_computer/src/stateful/address_cohort.rs b/crates/brk_computer/src/stateful/address_cohort.rs index e8e12e72f..843784e2b 100644 --- a/crates/brk_computer/src/stateful/address_cohort.rs +++ b/crates/brk_computer/src/stateful/address_cohort.rs @@ -1,16 +1,17 @@ use std::{ops::Deref, path::Path, sync::Arc}; -use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; +use brk_structs::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, VecIterator, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File, + Format, GenericStoredVec, VecIterator, }; use crate::{ - Indexes, fetched, + Indexes, grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, - indexes, market, + indexes, market, price, stateful::{ common, r#trait::{CohortVecs, DynCohortVecs}, @@ -28,8 +29,8 @@ pub struct Vecs { pub inner: common::Vecs, - pub height_to_address_count: EagerVec, - pub indexes_to_address_count: ComputedVecsFromHeight, + pub height_to_address_count: EagerVec, + pub indexes_to_address_count: ComputedVecsFromHeight, } impl Vecs { @@ -41,11 +42,11 @@ impl Vecs { format: Format, version: Version, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, states_path: &Path, compute_relative_to_all: bool, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); + ) -> Result { + let compute_dollars = price.is_some(); let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}")); @@ -79,7 +80,7 @@ impl Vecs { format, version, indexes, - fetched, + price, compute_relative_to_all, false, )?, @@ -170,10 +171,10 @@ impl DynCohortVecs for Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.indexes_to_address_count.compute_rest( indexes, starting_indexes, @@ -182,7 +183,7 @@ impl DynCohortVecs for Vecs { )?; self.inner - .compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + .compute_rest_part1(indexer, indexes, price, starting_indexes, exit) } fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { @@ -223,7 +224,7 @@ impl CohortVecs for Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, market: &market::Vecs, height_to_supply: &impl AnyIterableVec, @@ -231,11 +232,11 @@ impl CohortVecs for Vecs { height_to_realized_cap: Option<&impl AnyIterableVec>, dateindex_to_realized_cap: Option<&impl AnyIterableVec>, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.inner.compute_rest_part2( indexer, indexes, - fetched, + price, starting_indexes, market, height_to_supply, diff --git a/crates/brk_computer/src/stateful/address_cohorts.rs b/crates/brk_computer/src/stateful/address_cohorts.rs index c1cdaad6b..2350326b4 100644 --- a/crates/brk_computer/src/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/stateful/address_cohorts.rs @@ -1,16 +1,16 @@ use std::{path::Path, sync::Arc}; -use brk_core::{ - AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex, - Dollars, GroupFilter, Height, Result, Version, -}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyIterableVec, Computation, File, Format}; +use brk_structs::{ + AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex, + Dollars, GroupFilter, Height, Version, +}; +use brk_vecs::{AnyIterableVec, Computation, Exit, File, Format}; use derive_deref::{Deref, DerefMut}; use crate::{ - Indexes, fetched, indexes, market, + Indexes, indexes, market, price, stateful::{ address_cohort, r#trait::{CohortVecs, DynCohortVecs}, @@ -29,9 +29,9 @@ impl Vecs { _computation: Computation, format: Format, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, states_path: &Path, - ) -> color_eyre::Result { + ) -> Result { Ok(Self( AddressGroups { amount_range: ByAmountRange { @@ -42,7 +42,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -53,7 +53,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -64,7 +64,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -75,7 +75,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -86,7 +86,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -97,7 +97,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -108,7 +108,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -119,7 +119,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -130,7 +130,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -141,7 +141,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -152,7 +152,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -163,7 +163,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -174,7 +174,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -185,7 +185,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -196,7 +196,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -209,7 +209,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -220,7 +220,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -231,7 +231,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -242,7 +242,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -253,7 +253,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -264,7 +264,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -275,7 +275,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -286,7 +286,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -297,7 +297,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -308,7 +308,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -319,7 +319,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -330,7 +330,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -341,7 +341,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -354,7 +354,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -365,7 +365,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -376,7 +376,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -387,7 +387,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -398,7 +398,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -409,7 +409,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -420,7 +420,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -431,7 +431,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -442,7 +442,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -453,7 +453,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -464,7 +464,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -475,7 +475,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -486,7 +486,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, )?, @@ -546,12 +546,12 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.as_mut_vecs().into_iter().try_for_each(|(_, v)| { - v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + v.compute_rest_part1(indexer, indexes, price, starting_indexes, exit) }) } @@ -560,7 +560,7 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, market: &market::Vecs, height_to_supply: &impl AnyIterableVec, @@ -568,13 +568,13 @@ impl Vecs { height_to_realized_cap: Option<&impl AnyIterableVec>, dateindex_to_realized_cap: Option<&impl AnyIterableVec>, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.0.as_boxed_mut_vecs().into_iter().try_for_each(|v| { v.into_iter().try_for_each(|(_, v)| { v.compute_rest_part2( indexer, indexes, - fetched, + price, starting_indexes, market, height_to_supply, diff --git a/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs index 1424c9cc6..54a6bf313 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs @@ -1,11 +1,11 @@ -use brk_core::{ByAddressType, Height}; +use brk_structs::{ByAddressType, Height}; use brk_vecs::VecIterator; use derive_deref::{Deref, DerefMut}; use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount; #[derive(Debug, Default, Deref, DerefMut)] -pub struct AddressTypeToAddressCount(ByAddressType); +pub struct AddressTypeToAddressCount(ByAddressType); impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount { fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self { diff --git a/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs index 8ec38ba83..8273634aa 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs @@ -1,15 +1,15 @@ -use brk_core::{ByAddressType, Height, Result, StoredUsize}; -use brk_exit::Exit; -use brk_vecs::EagerVec; +use brk_error::Result; +use brk_structs::{ByAddressType, Height, StoredU64}; +use brk_vecs::{EagerVec, Exit, GenericStoredVec}; use derive_deref::{Deref, DerefMut}; use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount; #[derive(Debug, Clone, Deref, DerefMut)] -pub struct AddressTypeToHeightToAddressCount(ByAddressType>); +pub struct AddressTypeToHeightToAddressCount(ByAddressType>); -impl From>> for AddressTypeToHeightToAddressCount { - fn from(value: ByAddressType>) -> Self { +impl From>> for AddressTypeToHeightToAddressCount { + fn from(value: ByAddressType>) -> Self { Self(value) } } diff --git a/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs index 89746fab2..e4eb38c73 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs @@ -1,6 +1,6 @@ -use brk_core::{ByAddressType, StoredUsize}; -use brk_exit::Exit; -use brk_vecs::AnyCollectableVec; +use brk_error::Result; +use brk_structs::{ByAddressType, StoredU64}; +use brk_vecs::{AnyCollectableVec, Exit}; use derive_deref::{Deref, DerefMut}; use crate::{ @@ -9,12 +9,10 @@ use crate::{ }; #[derive(Clone, Deref, DerefMut)] -pub struct AddressTypeToIndexesToAddressCount(ByAddressType>); +pub struct AddressTypeToIndexesToAddressCount(ByAddressType>); -impl From>> - for AddressTypeToIndexesToAddressCount -{ - fn from(value: ByAddressType>) -> Self { +impl From>> for AddressTypeToIndexesToAddressCount { + fn from(value: ByAddressType>) -> Self { Self(value) } } @@ -27,7 +25,7 @@ impl AddressTypeToIndexesToAddressCount { starting_indexes: &Indexes, exit: &Exit, addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.p2pk65.compute_rest( indexes, starting_indexes, diff --git a/crates/brk_computer/src/stateful/addresstype_to_typeindex_set.rs b/crates/brk_computer/src/stateful/addresstype_to_typeindex_set.rs index dd5f49b37..4c03fa656 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_typeindex_set.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_typeindex_set.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; -use brk_core::TypeIndex; +use brk_structs::TypeIndex; use derive_deref::{Deref, DerefMut}; use super::ByAddressType; diff --git a/crates/brk_computer/src/stateful/addresstype_to_typeindex_tree.rs b/crates/brk_computer/src/stateful/addresstype_to_typeindex_tree.rs index 960d3b691..c2ba8d35f 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_typeindex_tree.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_typeindex_tree.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, mem}; -use brk_core::TypeIndex; +use brk_structs::TypeIndex; use derive_deref::{Deref, DerefMut}; use super::ByAddressType; diff --git a/crates/brk_computer/src/stateful/common.rs b/crates/brk_computer/src/stateful/common.rs index c8bba9681..f9b43db91 100644 --- a/crates/brk_computer/src/stateful/common.rs +++ b/crates/brk_computer/src/stateful/common.rs @@ -1,22 +1,22 @@ use std::sync::Arc; -use brk_core::{ - Bitcoin, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredF64, StoredUsize, Version, -}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; +use brk_structs::{ + Bitcoin, DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64, Version, +}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec, - File, Format, VecIterator, + AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, + EagerVec, Exit, File, Format, GenericStoredVec, VecIterator, }; use crate::{ - Indexes, fetched, + Indexes, grouped::{ ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions, }, - indexes, market, + indexes, market, price, states::CohortState, }; @@ -27,7 +27,7 @@ pub struct Vecs { // Cumulative pub height_to_realized_cap: Option>, pub height_to_supply: EagerVec, - pub height_to_utxo_count: EagerVec, + pub height_to_utxo_count: EagerVec, // Single pub dateindex_to_supply_even: Option>, pub dateindex_to_supply_in_loss: Option>, @@ -68,7 +68,7 @@ pub struct Vecs { pub indexes_to_realized_value: Option>, pub height_to_supply_value: ComputedHeightValueVecs, pub indexes_to_supply: ComputedValueVecsFromDateIndex, - pub indexes_to_utxo_count: ComputedVecsFromHeight, + pub indexes_to_utxo_count: ComputedVecsFromHeight, pub indexes_to_value_created: Option>, pub indexes_to_value_destroyed: Option>, pub indexes_to_unrealized_profit: Option>, @@ -134,11 +134,11 @@ impl Vecs { format: Format, version: Version, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, compute_relative_to_all: bool, ratio_extended: bool, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); + ) -> Result { + let compute_dollars = price.is_some(); // let prefix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{s}_{name}")); @@ -1368,7 +1368,7 @@ impl Vecs { self.height_to_utxo_count.forced_push_at( height, - StoredUsize::from(state.supply.utxos), + StoredU64::from(state.supply.utxos), exit, )?; @@ -1899,12 +1899,12 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.height_to_supply_value.compute_rest( - fetched, + price, starting_indexes, exit, Some(&self.height_to_supply), @@ -1913,7 +1913,7 @@ impl Vecs { self.indexes_to_supply.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |v, _, indexes, starting_indexes, exit| { @@ -1925,14 +1925,15 @@ impl Vecs { &indexes.dateindex_to_first_height, |(i, height, ..)| { let count = dateindex_to_height_count_iter.unwrap_get_inner(i); - if count == StoredUsize::default() { + if count == StoredU64::default() { unreachable!() } let supply = height_to_supply_iter.unwrap_get_inner(height + (*count - 1)); (i, supply) }, exit, - ) + )?; + Ok(()) }, )?; @@ -1946,7 +1947,7 @@ impl Vecs { self.height_to_halved_supply_value.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |v, _, _, starting_indexes, exit| { @@ -1955,14 +1956,15 @@ impl Vecs { &self.height_to_supply, |(h, v, ..)| (h, v / 2), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_halved_supply.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |v, _, _, starting_indexes, exit| { @@ -1971,7 +1973,8 @@ impl Vecs { self.indexes_to_supply.sats.dateindex.as_ref().unwrap(), |(i, sats, ..)| (i, sats / 2), exit, - ) + )?; + Ok(()) }, )?; @@ -1986,7 +1989,8 @@ impl Vecs { &self.height_to_satblocks_destroyed, |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), exit, - ) + )?; + Ok(()) }, )?; @@ -2001,7 +2005,8 @@ impl Vecs { &self.height_to_satdays_destroyed, |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), exit, - ) + )?; + Ok(()) }, )?; @@ -2013,7 +2018,7 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, market: &market::Vecs, height_to_supply: &impl AnyIterableVec, @@ -2021,7 +2026,7 @@ impl Vecs { height_to_realized_cap: Option<&impl AnyIterableVec>, dateindex_to_realized_cap: Option<&impl AnyIterableVec>, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if let Some(v) = self .indexes_to_supply_relative_to_circulating_supply .as_mut() @@ -2037,7 +2042,8 @@ impl Vecs { &self.height_to_supply_value.bitcoin, height_to_supply, exit, - ) + )?; + Ok(()) }, )?; } @@ -2064,7 +2070,8 @@ impl Vecs { self.height_to_realized_cap.as_ref().unwrap(), &self.height_to_supply_value.bitcoin, exit, - ) + )?; + Ok(()) }, )?; @@ -2074,7 +2081,7 @@ impl Vecs { .compute_rest( indexer, indexes, - fetched.as_ref().unwrap(), + price.as_ref().unwrap(), starting_indexes, exit, Some( @@ -2120,7 +2127,8 @@ impl Vecs { self.height_to_realized_loss.as_ref().unwrap(), |(i, v, ..)| (i, v * -1_i64), exit, - ) + )?; + Ok(()) }, )?; @@ -2182,7 +2190,8 @@ impl Vecs { .unwrap_last(), 30, exit, - ) + )?; + Ok(()) }, )?; @@ -2200,7 +2209,8 @@ impl Vecs { self.height_to_realized_profit.as_ref().unwrap(), self.height_to_realized_loss.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2218,7 +2228,8 @@ impl Vecs { self.height_to_realized_profit.as_ref().unwrap(), self.height_to_realized_loss.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2282,7 +2293,7 @@ impl Vecs { .compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(self.dateindex_to_supply_in_profit.as_ref().unwrap()), @@ -2293,7 +2304,7 @@ impl Vecs { .compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(self.dateindex_to_supply_in_loss.as_ref().unwrap()), @@ -2301,7 +2312,7 @@ impl Vecs { self.indexes_to_supply_even.as_mut().unwrap().compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(self.dateindex_to_supply_even.as_ref().unwrap()), @@ -2367,7 +2378,8 @@ impl Vecs { self.dateindex_to_unrealized_loss.as_ref().unwrap(), |(h, v, ..)| (h, v * -1_i64), exit, - ) + )?; + Ok(()) }, )?; self.height_to_net_unrealized_profit_and_loss @@ -2394,7 +2406,8 @@ impl Vecs { self.dateindex_to_unrealized_profit.as_ref().unwrap(), self.dateindex_to_unrealized_loss.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.height_to_net_unrealized_profit_and_loss_relative_to_market_cap @@ -2427,7 +2440,8 @@ impl Vecs { .unwrap(), market.indexes_to_marketcap.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2445,7 +2459,8 @@ impl Vecs { self.height_to_realized_profit.as_ref().unwrap(), *height_to_realized_cap.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2463,7 +2478,8 @@ impl Vecs { self.height_to_realized_loss.as_ref().unwrap(), *height_to_realized_cap.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2486,7 +2502,8 @@ impl Vecs { .unwrap(), *height_to_realized_cap.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2494,7 +2511,7 @@ impl Vecs { .as_mut() .unwrap() .compute_rest( - fetched, + price, starting_indexes, exit, Some(self.height_to_supply_even.as_ref().unwrap()), @@ -2503,7 +2520,7 @@ impl Vecs { .as_mut() .unwrap() .compute_rest( - fetched, + price, starting_indexes, exit, Some(self.height_to_supply_in_loss.as_ref().unwrap()), @@ -2512,7 +2529,7 @@ impl Vecs { .as_mut() .unwrap() .compute_rest( - fetched, + price, starting_indexes, exit, Some(self.height_to_supply_in_profit.as_ref().unwrap()), @@ -2572,7 +2589,8 @@ impl Vecs { .unwrap(), self.indexes_to_supply.bitcoin.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_supply_in_loss_relative_to_own_supply @@ -2595,7 +2613,8 @@ impl Vecs { .unwrap(), self.indexes_to_supply.bitcoin.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_supply_in_profit_relative_to_own_supply @@ -2618,7 +2637,8 @@ impl Vecs { .unwrap(), self.indexes_to_supply.bitcoin.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2640,7 +2660,8 @@ impl Vecs { .unwrap_cumulative(), 30, exit, - ) + )?; + Ok(()) }, )?; @@ -2658,7 +2679,8 @@ impl Vecs { self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(), *dateindex_to_realized_cap.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2676,7 +2698,8 @@ impl Vecs { self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(), market.indexes_to_marketcap.dateindex.as_ref().unwrap(), exit, - ) + )?; + Ok(()) }, )?; @@ -2736,7 +2759,8 @@ impl Vecs { .unwrap(), dateindex_to_supply, exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_supply_in_loss_relative_to_circulating_supply @@ -2759,7 +2783,8 @@ impl Vecs { .unwrap(), dateindex_to_supply, exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_supply_in_profit_relative_to_circulating_supply @@ -2782,7 +2807,8 @@ impl Vecs { .unwrap(), dateindex_to_supply, exit, - ) + )?; + Ok(()) }, )?; } diff --git a/crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs b/crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs index 65449bc63..3a3b921df 100644 --- a/crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs +++ b/crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use brk_core::Height; +use brk_structs::Height; use derive_deref::{Deref, DerefMut}; use crate::stateful::AddressTypeToVec; diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index d70be1075..9d76341e0 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -1,31 +1,26 @@ use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, sync::Arc, thread}; -use brk_core::{ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_structs::{ AnyAddressDataIndexEnum, AnyAddressIndex, ByAddressType, ByAnyAddress, CheckedSub, DateIndex, Dollars, EmptyAddressData, EmptyAddressIndex, Height, InputIndex, LoadedAddressData, LoadedAddressIndex, OutputIndex, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, - P2WSHAddressIndex, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version, + P2WSHAddressIndex, Sats, StoredU64, Timestamp, TypeIndex, Version, }; -use brk_exit::Exit; -use brk_indexer::Indexer; use brk_vecs::{ - AnyCollectableVec, AnyStampedVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format, - GenericStoredVec, Reader, Stamp, StampedVec, StoredIndex, StoredVec, VecIterator, + AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, Exit, File, + Format, GenericStoredVec, RawVec, Reader, Stamp, StoredIndex, VecIterator, }; use log::info; use rayon::prelude::*; use crate::{ - BlockState, SupplyState, Transacted, - grouped::{ComputedVecsFromHeight, Source}, - market, -}; - -use super::{ - Indexes, fetched, + BlockState, Indexes, SupplyState, Transacted, grouped::{ComputedValueVecsFromHeight, VecBuilderOptions}, - indexes, transactions, + grouped::{ComputedVecsFromHeight, Source}, + indexes, market, price, transactions, }; mod address_cohort; @@ -58,7 +53,9 @@ const VERSION: Version = Version::new(21); #[derive(Clone)] pub struct Vecs { - pub chain_state: StoredVec, + file: Arc, + + pub chain_state: RawVec, pub height_to_unspendable_supply: EagerVec, pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, @@ -71,51 +68,52 @@ pub struct Vecs { pub utxo_cohorts: utxo_cohorts::Vecs, pub address_cohorts: address_cohorts::Vecs, - pub p2pk33addressindex_to_anyaddressindex: StampedVec, - pub p2pk65addressindex_to_anyaddressindex: StampedVec, - pub p2pkhaddressindex_to_anyaddressindex: StampedVec, - pub p2shaddressindex_to_anyaddressindex: StampedVec, - pub p2traddressindex_to_anyaddressindex: StampedVec, - pub p2wpkhaddressindex_to_anyaddressindex: StampedVec, - pub p2wshaddressindex_to_anyaddressindex: StampedVec, - pub p2aaddressindex_to_anyaddressindex: StampedVec, - pub loadedaddressindex_to_loadedaddressdata: StampedVec, - pub emptyaddressindex_to_emptyaddressdata: StampedVec, + pub p2pk33addressindex_to_anyaddressindex: RawVec, + pub p2pk65addressindex_to_anyaddressindex: RawVec, + pub p2pkhaddressindex_to_anyaddressindex: RawVec, + pub p2shaddressindex_to_anyaddressindex: RawVec, + pub p2traddressindex_to_anyaddressindex: RawVec, + pub p2wpkhaddressindex_to_anyaddressindex: RawVec, + pub p2wshaddressindex_to_anyaddressindex: RawVec, + pub p2aaddressindex_to_anyaddressindex: RawVec, + pub loadedaddressindex_to_loadedaddressdata: RawVec, + pub emptyaddressindex_to_emptyaddressdata: RawVec, - pub indexes_to_address_count: ComputedVecsFromHeight, - pub indexes_to_empty_address_count: ComputedVecsFromHeight, + pub indexes_to_address_count: ComputedVecsFromHeight, + pub indexes_to_empty_address_count: ComputedVecsFromHeight, } impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, states_path: &Path, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); + ) -> Result { + let file = Arc::new(File::open(&parent.join("stateful"))?); - let chain_file = Arc::new(File::open(&states_path.join("chain"))?); + let compute_dollars = price.is_some(); + + let chain_file = Arc::new(File::open(&parent.join("chain"))?); Ok(Self { - chain_state: StoredVec::forced_import( + chain_state: RawVec::forced_import( &chain_file, "chain", version + VERSION + Version::ZERO, - Format::Raw, )?, height_to_unspendable_supply: EagerVec::forced_import( - file, + &file, "unspendable_supply", version + VERSION + Version::ZERO, format, )?, indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( - file, + &file, "unspendable_supply", Source::None, version + VERSION + Version::ZERO, @@ -126,13 +124,13 @@ impl Vecs { indexes, )?, height_to_opreturn_supply: EagerVec::forced_import( - file, + &file, "opreturn_supply", version + VERSION + Version::ZERO, format, )?, indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import( - file, + &file, "opreturn_supply", Source::None, version + VERSION + Version::ZERO, @@ -143,7 +141,7 @@ impl Vecs { indexes, )?, indexes_to_address_count: ComputedVecsFromHeight::forced_import( - file, + &file, "address_count", Source::Compute, version + VERSION + Version::ZERO, @@ -153,7 +151,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import( - file, + &file, "empty_address_count", Source::Compute, version + VERSION + Version::ZERO, @@ -165,49 +163,49 @@ impl Vecs { addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { p2pk65: EagerVec::forced_import( - file, + &file, "p2pk65_address_count", version + VERSION + Version::ZERO, format, )?, p2pk33: EagerVec::forced_import( - file, + &file, "p2pk33_address_count", version + VERSION + Version::ZERO, format, )?, p2pkh: EagerVec::forced_import( - file, + &file, "p2pkh_address_count", version + VERSION + Version::ZERO, format, )?, p2sh: EagerVec::forced_import( - file, + &file, "p2sh_address_count", version + VERSION + Version::ZERO, format, )?, p2wpkh: EagerVec::forced_import( - file, + &file, "p2wpkh_address_count", version + VERSION + Version::ZERO, format, )?, p2wsh: EagerVec::forced_import( - file, + &file, "p2wsh_address_count", version + VERSION + Version::ZERO, format, )?, p2tr: EagerVec::forced_import( - file, + &file, "p2tr_address_count", version + VERSION + Version::ZERO, format, )?, p2a: EagerVec::forced_import( - file, + &file, "p2a_address_count", version + VERSION + Version::ZERO, format, @@ -217,49 +215,49 @@ impl Vecs { addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { p2pk65: EagerVec::forced_import( - file, + &file, "p2pk65_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2pk33: EagerVec::forced_import( - file, + &file, "p2pk33_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2pkh: EagerVec::forced_import( - file, + &file, "p2pkh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2sh: EagerVec::forced_import( - file, + &file, "p2sh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2wpkh: EagerVec::forced_import( - file, + &file, "p2wpkh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2wsh: EagerVec::forced_import( - file, + &file, "p2wsh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2tr: EagerVec::forced_import( - file, + &file, "p2tr_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2a: EagerVec::forced_import( - file, + &file, "p2a_empty_address_count", version + VERSION + Version::ZERO, format, @@ -269,7 +267,7 @@ impl Vecs { addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from( ByAddressType { p2pk65: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pk65_address_count", Source::None, version + VERSION + Version::ZERO, @@ -279,7 +277,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pk33: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pk33_address_count", Source::None, version + VERSION + Version::ZERO, @@ -289,7 +287,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pkh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pkh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -299,7 +297,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2sh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2sh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -309,7 +307,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wpkh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2wpkh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -319,7 +317,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wsh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2wsh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -329,7 +327,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2tr: ComputedVecsFromHeight::forced_import( - file, + &file, "p2tr_address_count", Source::None, version + VERSION + Version::ZERO, @@ -339,7 +337,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2a: ComputedVecsFromHeight::forced_import( - file, + &file, "p2a_address_count", Source::None, version + VERSION + Version::ZERO, @@ -353,7 +351,7 @@ impl Vecs { addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from( ByAddressType { p2pk65: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pk65_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -363,7 +361,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pk33: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pk33_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -373,7 +371,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pkh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pkh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -383,7 +381,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2sh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2sh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -393,7 +391,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wpkh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2wpkh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -403,7 +401,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wsh: ComputedVecsFromHeight::forced_import( - file, + &file, "p2wsh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -413,7 +411,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2tr: ComputedVecsFromHeight::forced_import( - file, + &file, "p2tr_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -423,7 +421,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2a: ComputedVecsFromHeight::forced_import( - file, + &file, "p2a_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -435,85 +433,77 @@ impl Vecs { }, ), utxo_cohorts: utxo_cohorts::Vecs::forced_import( - file, + &file, version, computation, format, indexes, - fetched, + price, states_path, )?, address_cohorts: address_cohorts::Vecs::forced_import( - file, + &file, version, computation, format, indexes, - fetched, + price, states_path, )?, - p2aaddressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2aaddressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2pk33addressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2pk33addressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2pk65addressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2pk65addressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2pkhaddressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2pkhaddressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2shaddressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2shaddressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2traddressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2traddressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2wpkhaddressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2wpkhaddressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2wshaddressindex_to_anyaddressindex: StampedVec::forced_import( - file, + p2wshaddressindex_to_anyaddressindex: RawVec::forced_import( + &file, "anyaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - loadedaddressindex_to_loadedaddressdata: StampedVec::forced_import( - file, + loadedaddressindex_to_loadedaddressdata: RawVec::forced_import( + &file, "loadedaddressdata", version + VERSION + Version::ZERO, - Format::Raw, )?, - emptyaddressindex_to_emptyaddressdata: StampedVec::forced_import( - file, + emptyaddressindex_to_emptyaddressdata: RawVec::forced_import( + &file, "emptyaddressdata", version + VERSION + Version::ZERO, - Format::Raw, )?, + + file, }) } @@ -523,12 +513,12 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, transactions: &transactions::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, market: &market::Vecs, // Must take ownership as its indexes will be updated for this specific function starting_indexes: &mut Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { let height_to_first_outputindex = &indexer.vecs.height_to_first_outputindex; let height_to_first_inputindex = &indexer.vecs.height_to_first_inputindex; let height_to_first_p2aaddressindex = &indexer.vecs.height_to_first_p2aaddressindex; @@ -554,12 +544,12 @@ impl Vecs { .height .as_ref() .unwrap(); - let height_to_close = fetched + let height_to_close = price .as_ref() - .map(|fetched| &fetched.chainindexes_to_close.height); - let dateindex_to_close = fetched + .map(|price| &price.chainindexes_to_close.height); + let dateindex_to_close = price .as_ref() - .map(|fetched| fetched.timeindexes_to_close.dateindex.as_ref().unwrap()); + .map(|price| price.timeindexes_to_close.dateindex.as_ref().unwrap()); let height_to_date_fixed = &indexes.height_to_date_fixed; let dateindex_to_first_height = &indexes.dateindex_to_first_height; let dateindex_to_height_count = &indexes.dateindex_to_height_count; @@ -632,56 +622,16 @@ impl Vecs { .unwrap_or_default(), ) .min(chain_state_starting_height) - .min( - Height::from(u64::from( - self.p2pk33addressindex_to_anyaddressindex.stamp(), - )) - .incremented(), - ) - .min( - Height::from(u64::from( - self.p2pk65addressindex_to_anyaddressindex.stamp(), - )) - .incremented(), - ) - .min( - Height::from(u64::from(self.p2pkhaddressindex_to_anyaddressindex.stamp())) - .incremented(), - ) - .min( - Height::from(u64::from(self.p2shaddressindex_to_anyaddressindex.stamp())) - .incremented(), - ) - .min( - Height::from(u64::from(self.p2traddressindex_to_anyaddressindex.stamp())) - .incremented(), - ) - .min( - Height::from(u64::from( - self.p2wpkhaddressindex_to_anyaddressindex.stamp(), - )) - .incremented(), - ) - .min( - Height::from(u64::from(self.p2wshaddressindex_to_anyaddressindex.stamp())) - .incremented(), - ) - .min( - Height::from(u64::from(self.p2aaddressindex_to_anyaddressindex.stamp())) - .incremented(), - ) - .min( - Height::from(u64::from( - self.loadedaddressindex_to_loadedaddressdata.stamp(), - )) - .incremented(), - ) - .min( - Height::from(u64::from( - self.emptyaddressindex_to_emptyaddressdata.stamp(), - )) - .incremented(), - ) + .min(Height::from(self.p2pk33addressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.p2pk65addressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.p2pkhaddressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.p2shaddressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.p2traddressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.p2wpkhaddressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.p2wshaddressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.p2aaddressindex_to_anyaddressindex.stamp()).incremented()) + .min(Height::from(self.loadedaddressindex_to_loadedaddressdata.stamp()).incremented()) + .min(Height::from(self.emptyaddressindex_to_emptyaddressdata.stamp()).incremented()) .min(Height::from(self.height_to_unspendable_supply.len())) .min(Height::from(self.height_to_opreturn_supply.len())) .cmp(&chain_state_starting_height) @@ -745,7 +695,7 @@ impl Vecs { .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; }; - let last_height = Height::from(u64::from(indexer.vecs.height_to_blockhash.stamp())); + let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp()); if starting_height <= last_height { let inputindex_to_outputindex_reader = inputindex_to_outputindex.create_reader(); @@ -836,7 +786,7 @@ impl Vecs { (height.unwrap_to_usize()..height_to_date_fixed.len()) .map(Height::from) - .try_for_each(|_height| -> color_eyre::Result<()> { + .try_for_each(|_height| -> Result<()> { height = _height; info!("Processing chain at {height}..."); @@ -903,7 +853,7 @@ impl Vecs { .tick_tock_next_block(&chain_state, timestamp); }); - let (transacted, addresstype_to_typedindex_to_received_data, receiving_addresstype_to_typeindex_to_addressdatawithsource) = (first_outputindex..first_outputindex + *output_count) + let (transacted, addresstype_to_typedindex_to_received_data, receiving_addresstype_to_typeindex_to_addressdatawithsource) = (first_outputindex..first_outputindex + usize::from(output_count)) .into_par_iter() .map(OutputIndex::from) .map(|outputindex| { @@ -987,7 +937,7 @@ impl Vecs { ); // Skip coinbase - let (height_to_sent, addresstype_to_typedindex_to_sent_data, sending_addresstype_to_typeindex_to_addressdatawithsource) = (first_inputindex + 1..first_inputindex + *input_count) + let (height_to_sent, addresstype_to_typedindex_to_sent_data, sending_addresstype_to_typeindex_to_addressdatawithsource) = (first_inputindex + 1..first_inputindex + usize::from(input_count)) .into_par_iter() .map(InputIndex::from) .map(|inputindex| { @@ -1287,7 +1237,8 @@ impl Vecs { .map(|(_, v)| v) .collect::>(), exit, - ) + )?; + Ok(()) }, )?; @@ -1306,14 +1257,15 @@ impl Vecs { .map(|(_, v)| v) .collect::>(), exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_unspendable_supply.compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(&self.height_to_unspendable_supply), @@ -1321,7 +1273,7 @@ impl Vecs { self.indexes_to_opreturn_supply.compute_rest( indexer, indexes, - fetched, + price, starting_indexes, exit, Some(&self.height_to_opreturn_supply), @@ -1342,15 +1294,10 @@ impl Vecs { )?; self.utxo_cohorts - .compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)?; + .compute_rest_part1(indexer, indexes, price, starting_indexes, exit)?; - self.address_cohorts.compute_rest_part1( - indexer, - indexes, - fetched, - starting_indexes, - exit, - )?; + self.address_cohorts + .compute_rest_part1(indexer, indexes, price, starting_indexes, exit)?; info!("Computing rest part 2..."); @@ -1384,7 +1331,7 @@ impl Vecs { self.utxo_cohorts.compute_rest_part2( indexer, indexes, - fetched, + price, starting_indexes, market, height_to_supply, @@ -1397,7 +1344,7 @@ impl Vecs { self.address_cohorts.compute_rest_part2( indexer, indexes, - fetched, + price, starting_indexes, market, height_to_supply, @@ -1407,6 +1354,8 @@ impl Vecs { exit, )?; + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } @@ -1423,16 +1372,16 @@ impl Vecs { >, addresstypeindex_to_anyaddressindex_reader_opt: &ByAddressType>, anyaddressindex_to_anyaddressdata_reader_opt: &ByAnyAddress>, - p2pk33addressindex_to_anyaddressindex: &StampedVec, - p2pk65addressindex_to_anyaddressindex: &StampedVec, - p2pkhaddressindex_to_anyaddressindex: &StampedVec, - p2shaddressindex_to_anyaddressindex: &StampedVec, - p2traddressindex_to_anyaddressindex: &StampedVec, - p2wpkhaddressindex_to_anyaddressindex: &StampedVec, - p2wshaddressindex_to_anyaddressindex: &StampedVec, - p2aaddressindex_to_anyaddressindex: &StampedVec, - loadedaddressindex_to_loadedaddressdata: &StampedVec, - emptyaddressindex_to_emptyaddressdata: &StampedVec, + p2pk33addressindex_to_anyaddressindex: &RawVec, + p2pk65addressindex_to_anyaddressindex: &RawVec, + p2pkhaddressindex_to_anyaddressindex: &RawVec, + p2shaddressindex_to_anyaddressindex: &RawVec, + p2traddressindex_to_anyaddressindex: &RawVec, + p2wpkhaddressindex_to_anyaddressindex: &RawVec, + p2wshaddressindex_to_anyaddressindex: &RawVec, + p2aaddressindex_to_anyaddressindex: &RawVec, + loadedaddressindex_to_loadedaddressdata: &RawVec, + emptyaddressindex_to_emptyaddressdata: &RawVec, ) -> Option> { if *first_addressindexes.get(address_type).unwrap() <= typeindex { return Some(WithAddressDataSource::New(LoadedAddressData::default())); @@ -1643,7 +1592,8 @@ impl Vecs { emptyaddressdata, )) => self .emptyaddressindex_to_emptyaddressdata - .update(emptyaddressindex, emptyaddressdata), + .update(emptyaddressindex, emptyaddressdata) + .map_err(|e| e.into()), WithAddressDataSource::FromLoadedAddressDataVec(( loadedaddressindex, emptyaddressdata, @@ -1695,7 +1645,8 @@ impl Vecs { loadedaddressdata, )) => self .loadedaddressindex_to_loadedaddressdata - .update(loadedaddressindex, loadedaddressdata), + .update(loadedaddressindex, loadedaddressdata) + .map_err(|e| e.into()), WithAddressDataSource::FromEmptyAddressDataVec(( emptyaddressindex, loadedaddressdata, @@ -1753,30 +1704,31 @@ impl Vecs { .p2aaddressindex_to_anyaddressindex .update_or_push(typeindex.into(), anyaddressindex), _ => unreachable!(), - } + }?; + Ok(()) }) })?; self.p2pk33addressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.p2pk65addressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.p2pkhaddressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.p2shaddressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.p2traddressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.p2wpkhaddressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.p2wshaddressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.p2aaddressindex_to_anyaddressindex - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.loadedaddressindex_to_loadedaddressdata - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.emptyaddressindex_to_emptyaddressdata - .flush(Stamp::from(u64::from(height)))?; + .stamped_flush(Stamp::from(height))?; self.chain_state.truncate_if_needed(Height::ZERO)?; chain_state.iter().for_each(|block_state| { @@ -1838,8 +1790,8 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { WithAddressDataSource, >, price: Option, - addresstype_to_address_count: &mut ByAddressType, - addresstype_to_empty_address_count: &mut ByAddressType, + addresstype_to_address_count: &mut ByAddressType, + addresstype_to_empty_address_count: &mut ByAddressType, stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexTree< WithAddressDataSource, >, @@ -1928,9 +1880,9 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { WithAddressDataSource, >, price: Option, - addresstype_to_address_count: &mut ByAddressType, - addresstype_to_empty_address_count: &mut ByAddressType, - height_to_close_vec: Option<&Vec>>, + addresstype_to_address_count: &mut ByAddressType, + addresstype_to_empty_address_count: &mut ByAddressType, + height_to_close_vec: Option<&Vec>>, height_to_timestamp_fixed_vec: &[Timestamp], height: Height, timestamp: Timestamp, diff --git a/crates/brk_computer/src/stateful/range_map.rs b/crates/brk_computer/src/stateful/range_map.rs index ef4d20db4..70eac6e1a 100644 --- a/crates/brk_computer/src/stateful/range_map.rs +++ b/crates/brk_computer/src/stateful/range_map.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use brk_vecs::{StampedVec, StoredIndex, StoredType}; +use brk_vecs::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw}; #[derive(Debug)] pub struct RangeMap(BTreeMap); @@ -20,12 +20,26 @@ where } } -impl From<&StampedVec> for RangeMap +impl From<&RawVec> for RangeMap where I: StoredIndex, - T: StoredIndex + StoredType, + T: StoredIndex + StoredRaw, { - fn from(vec: &StampedVec) -> Self { + fn from(vec: &RawVec) -> Self { + Self( + vec.into_iter() + .map(|(i, v)| (v.into_owned(), i)) + .collect::>(), + ) + } +} + +impl From<&CompressedVec> for RangeMap +where + I: StoredIndex, + T: StoredIndex + StoredCompressed, +{ + fn from(vec: &CompressedVec) -> Self { Self( vec.into_iter() .map(|(i, v)| (v.into_owned(), i)) diff --git a/crates/brk_computer/src/stateful/trait.rs b/crates/brk_computer/src/stateful/trait.rs index fca7c6cce..6a44974dd 100644 --- a/crates/brk_computer/src/stateful/trait.rs +++ b/crates/brk_computer/src/stateful/trait.rs @@ -1,9 +1,9 @@ -use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, AnyIterableVec}; +use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version}; +use brk_vecs::{AnyCollectableVec, AnyIterableVec, Exit}; -use crate::{Indexes, fetched, indexes, market}; +use crate::{Indexes, indexes, market, price}; pub trait DynCohortVecs: Send + Sync { fn starting_height(&self) -> Height; @@ -30,10 +30,10 @@ pub trait DynCohortVecs: Send + Sync { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()>; + ) -> Result<()>; fn vecs(&self) -> Vec<&dyn AnyCollectableVec>; } @@ -51,7 +51,7 @@ pub trait CohortVecs: DynCohortVecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, market: &market::Vecs, height_to_supply: &impl AnyIterableVec, @@ -59,5 +59,5 @@ pub trait CohortVecs: DynCohortVecs { height_to_realized_cap: Option<&impl AnyIterableVec>, dateindex_to_realized_cap: Option<&impl AnyIterableVec>, exit: &Exit, - ) -> color_eyre::Result<()>; + ) -> Result<()>; } diff --git a/crates/brk_computer/src/stateful/utxo_cohort.rs b/crates/brk_computer/src/stateful/utxo_cohort.rs index 5bb562dd9..a0555a92d 100644 --- a/crates/brk_computer/src/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/stateful/utxo_cohort.rs @@ -1,12 +1,12 @@ use std::{ops::Deref, path::Path, sync::Arc}; -use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, File, Format}; +use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version}; +use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, Exit, File, Format}; use crate::{ - Indexes, UTXOCohortState, fetched, indexes, market, + Indexes, UTXOCohortState, indexes, market, price, stateful::{ common, r#trait::{CohortVecs, DynCohortVecs}, @@ -31,12 +31,12 @@ impl Vecs { format: Format, version: Version, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, states_path: &Path, compute_relative_to_all: bool, ratio_extended: bool, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); + ) -> Result { + let compute_dollars = price.is_some(); Ok(Self { starting_height: Height::ZERO, @@ -54,7 +54,7 @@ impl Vecs { format, version, indexes, - fetched, + price, compute_relative_to_all, ratio_extended, )?, @@ -123,12 +123,12 @@ impl DynCohortVecs for Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.inner - .compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + .compute_rest_part1(indexer, indexes, price, starting_indexes, exit) } fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { @@ -155,7 +155,7 @@ impl CohortVecs for Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, market: &market::Vecs, height_to_supply: &impl AnyIterableVec, @@ -163,11 +163,11 @@ impl CohortVecs for Vecs { height_to_realized_cap: Option<&impl AnyIterableVec>, dateindex_to_realized_cap: Option<&impl AnyIterableVec>, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.inner.compute_rest_part2( indexer, indexes, - fetched, + price, starting_indexes, market, height_to_supply, diff --git a/crates/brk_computer/src/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs index 54aba00a4..3ace57a9f 100644 --- a/crates/brk_computer/src/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -1,17 +1,17 @@ use std::{collections::BTreeMap, ops::ControlFlow, path::Path, sync::Arc}; -use brk_core::{ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_structs::{ Bitcoin, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, ByMaxAge, ByMinAge, BySpendableType, ByTerm, CheckedSub, DateIndex, Dollars, GroupFilter, HalvingEpoch, - Height, Result, Timestamp, UTXOGroups, Version, + Height, Timestamp, UTXOGroups, Version, }; -use brk_exit::Exit; -use brk_indexer::Indexer; -use brk_vecs::{AnyIterableVec, Computation, File, Format, StoredIndex}; +use brk_vecs::{AnyIterableVec, Computation, Exit, File, Format, StoredIndex}; use derive_deref::{Deref, DerefMut}; use crate::{ - Indexes, fetched, indexes, market, + Indexes, indexes, market, price, stateful::r#trait::DynCohortVecs, states::{BlockState, Transacted}, }; @@ -30,9 +30,9 @@ impl Vecs { _computation: Computation, format: Format, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, states_path: &Path, - ) -> color_eyre::Result { + ) -> Result { Ok(Self( UTXOGroups { all: utxo_cohort::Vecs::forced_import( @@ -42,7 +42,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, false, true, @@ -55,7 +55,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -67,7 +67,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -81,7 +81,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -93,7 +93,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -105,7 +105,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -117,7 +117,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -129,7 +129,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -143,7 +143,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -155,7 +155,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -167,7 +167,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -179,7 +179,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -191,7 +191,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -203,7 +203,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -215,7 +215,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -227,7 +227,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -239,7 +239,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -251,7 +251,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -263,7 +263,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -277,7 +277,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -289,7 +289,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -301,7 +301,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -313,7 +313,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -325,7 +325,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -337,7 +337,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -349,7 +349,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -361,7 +361,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -373,7 +373,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -385,7 +385,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -397,7 +397,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -409,7 +409,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -421,7 +421,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -433,7 +433,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -445,7 +445,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -457,7 +457,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -469,7 +469,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -481,7 +481,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -495,7 +495,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -507,7 +507,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -519,7 +519,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -531,7 +531,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -543,7 +543,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -555,7 +555,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -567,7 +567,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -579,7 +579,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -591,7 +591,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -603,7 +603,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -615,7 +615,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -627,7 +627,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -639,7 +639,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -651,7 +651,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -663,7 +663,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -675,7 +675,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -687,7 +687,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -699,7 +699,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -713,7 +713,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -725,7 +725,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -737,7 +737,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -749,7 +749,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -761,7 +761,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -773,7 +773,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -785,7 +785,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -797,7 +797,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -809,7 +809,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -821,7 +821,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -833,7 +833,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -845,7 +845,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -857,7 +857,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -869,7 +869,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -881,7 +881,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -893,7 +893,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -905,7 +905,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -917,7 +917,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -929,7 +929,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -941,7 +941,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, true, @@ -955,7 +955,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -967,7 +967,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -979,7 +979,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -991,7 +991,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1003,7 +1003,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1015,7 +1015,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1027,7 +1027,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1039,7 +1039,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1051,7 +1051,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1063,7 +1063,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1075,7 +1075,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1087,7 +1087,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1099,7 +1099,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1111,7 +1111,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1123,7 +1123,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1137,7 +1137,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1149,7 +1149,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1161,7 +1161,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1173,7 +1173,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1185,7 +1185,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1197,7 +1197,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1209,7 +1209,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1221,7 +1221,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1233,7 +1233,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1245,7 +1245,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1257,7 +1257,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1269,7 +1269,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1281,7 +1281,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1295,7 +1295,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1307,7 +1307,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1319,7 +1319,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1331,7 +1331,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1343,7 +1343,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1355,7 +1355,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1367,7 +1367,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1379,7 +1379,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1391,7 +1391,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1403,7 +1403,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1415,7 +1415,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1427,7 +1427,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1439,7 +1439,7 @@ impl Vecs { format, version + VERSION + Version::ZERO, indexes, - fetched, + price, states_path, true, false, @@ -1708,12 +1708,12 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.as_mut_vecs().into_iter().try_for_each(|(_, v)| { - v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + v.compute_rest_part1(indexer, indexes, price, starting_indexes, exit) }) } @@ -1722,7 +1722,7 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, starting_indexes: &Indexes, market: &market::Vecs, height_to_supply: &impl AnyIterableVec, @@ -1730,13 +1730,13 @@ impl Vecs { height_to_realized_cap: Option<&impl AnyIterableVec>, dateindex_to_realized_cap: Option<&impl AnyIterableVec>, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.0.as_boxed_mut_vecs().into_iter().try_for_each(|v| { v.into_iter().try_for_each(|(_, v)| { v.compute_rest_part2( indexer, indexes, - fetched, + price, starting_indexes, market, height_to_supply, diff --git a/crates/brk_computer/src/stateful/withaddressdatasource.rs b/crates/brk_computer/src/stateful/withaddressdatasource.rs index eee2ca157..c8af2d480 100644 --- a/crates/brk_computer/src/stateful/withaddressdatasource.rs +++ b/crates/brk_computer/src/stateful/withaddressdatasource.rs @@ -1,4 +1,4 @@ -use brk_core::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex}; +use brk_structs::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex}; #[derive(Debug)] pub enum WithAddressDataSource { diff --git a/crates/brk_computer/src/states/block.rs b/crates/brk_computer/src/states/block.rs index d4c371df5..27aaad538 100644 --- a/crates/brk_computer/src/states/block.rs +++ b/crates/brk_computer/src/states/block.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, SubAssign}; -use brk_core::{Dollars, Timestamp}; +use brk_structs::{Dollars, Timestamp}; use super::SupplyState; diff --git a/crates/brk_computer/src/states/cohorts/address.rs b/crates/brk_computer/src/states/cohorts/address.rs index 3778267e3..332d6e308 100644 --- a/crates/brk_computer/src/states/cohorts/address.rs +++ b/crates/brk_computer/src/states/cohorts/address.rs @@ -1,6 +1,7 @@ use std::path::Path; -use brk_core::{Dollars, Height, LoadedAddressData, Result, Sats}; +use brk_error::Result; +use brk_structs::{Dollars, Height, LoadedAddressData, Sats}; use crate::SupplyState; @@ -8,7 +9,7 @@ use super::CohortState; #[derive(Clone)] pub struct AddressCohortState { - pub address_count: usize, + pub address_count: u64, pub inner: CohortState, } @@ -47,14 +48,14 @@ impl AddressCohortState { let prev_realized_price = compute_price.then(|| addressdata.realized_price()); let prev_supply_state = SupplyState { - utxos: addressdata.outputs_len as usize, + utxos: addressdata.outputs_len as u64, value: addressdata.amount(), }; addressdata.send(value, prev_price)?; let supply_state = SupplyState { - utxos: addressdata.outputs_len as usize, + utxos: addressdata.outputs_len as u64, value: addressdata.amount(), }; @@ -82,14 +83,14 @@ impl AddressCohortState { let prev_realized_price = compute_price.then(|| address_data.realized_price()); let prev_supply_state = SupplyState { - utxos: address_data.outputs_len as usize, + utxos: address_data.outputs_len as u64, value: address_data.amount(), }; address_data.receive(value, price); let supply_state = SupplyState { - utxos: address_data.outputs_len as usize, + utxos: address_data.outputs_len as u64, value: address_data.amount(), }; diff --git a/crates/brk_computer/src/states/cohorts/common.rs b/crates/brk_computer/src/states/cohorts/common.rs index c2630e14f..00765512d 100644 --- a/crates/brk_computer/src/states/cohorts/common.rs +++ b/crates/brk_computer/src/states/cohorts/common.rs @@ -1,6 +1,7 @@ use std::{cmp::Ordering, path::Path}; -use brk_core::{CheckedSub, Dollars, Height, Result, Sats}; +use brk_error::Result; +use brk_structs::{CheckedSub, Dollars, Height, Sats}; use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState}; diff --git a/crates/brk_computer/src/states/cohorts/utxo.rs b/crates/brk_computer/src/states/cohorts/utxo.rs index 15ccd2836..d9a53068d 100644 --- a/crates/brk_computer/src/states/cohorts/utxo.rs +++ b/crates/brk_computer/src/states/cohorts/utxo.rs @@ -1,6 +1,7 @@ use std::path::Path; -use brk_core::{Height, Result}; +use brk_error::Result; +use brk_structs::Height; use derive_deref::{Deref, DerefMut}; use super::CohortState; diff --git a/crates/brk_computer/src/states/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs index 72eb9efe0..161712172 100644 --- a/crates/brk_computer/src/states/price_to_amount.rs +++ b/crates/brk_computer/src/states/price_to_amount.rs @@ -5,7 +5,8 @@ use std::{ path::{Path, PathBuf}, }; -use brk_core::{Dollars, Error, Height, Result, Sats}; +use brk_error::{Error, Result}; +use brk_structs::{Dollars, Height, Sats}; use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, IntoBytes}; diff --git a/crates/brk_computer/src/states/realized.rs b/crates/brk_computer/src/states/realized.rs index a4eea6f66..c804c9515 100644 --- a/crates/brk_computer/src/states/realized.rs +++ b/crates/brk_computer/src/states/realized.rs @@ -1,6 +1,6 @@ use std::cmp::Ordering; -use brk_core::{CheckedSub, Dollars}; +use brk_structs::{CheckedSub, Dollars}; use super::SupplyState; diff --git a/crates/brk_computer/src/states/supply.rs b/crates/brk_computer/src/states/supply.rs index c0e438a36..f94ed6fc0 100644 --- a/crates/brk_computer/src/states/supply.rs +++ b/crates/brk_computer/src/states/supply.rs @@ -1,12 +1,12 @@ use std::ops::{Add, AddAssign, SubAssign}; -use brk_core::{CheckedSub, LoadedAddressData, Sats}; +use brk_structs::{CheckedSub, LoadedAddressData, Sats}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] pub struct SupplyState { - pub utxos: usize, + pub utxos: u64, pub value: Sats, } @@ -43,7 +43,7 @@ impl SubAssign<&SupplyState> for SupplyState { impl From<&LoadedAddressData> for SupplyState { fn from(value: &LoadedAddressData) -> Self { Self { - utxos: value.outputs_len as usize, + utxos: value.outputs_len as u64, value: value.amount(), } } diff --git a/crates/brk_computer/src/states/transacted.rs b/crates/brk_computer/src/states/transacted.rs index 64f8c3b73..f48ed0921 100644 --- a/crates/brk_computer/src/states/transacted.rs +++ b/crates/brk_computer/src/states/transacted.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign}; -use brk_core::{ByAmountRange, GroupedByType, OutputType, Sats}; +use brk_structs::{ByAmountRange, GroupedByType, OutputType, Sats}; use super::SupplyState; diff --git a/crates/brk_computer/src/states/unrealized.rs b/crates/brk_computer/src/states/unrealized.rs index 51ba02520..f514295fc 100644 --- a/crates/brk_computer/src/states/unrealized.rs +++ b/crates/brk_computer/src/states/unrealized.rs @@ -1,4 +1,4 @@ -use brk_core::{Dollars, Sats}; +use brk_structs::{Dollars, Sats}; #[derive(Debug, Default, Clone)] pub struct UnrealizedState { diff --git a/crates/brk_computer/src/traits.rs b/crates/brk_computer/src/traits.rs new file mode 100644 index 000000000..5bff19d31 --- /dev/null +++ b/crates/brk_computer/src/traits.rs @@ -0,0 +1,316 @@ +use brk_error::Result; +use brk_structs::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32}; +use brk_vecs::{ + AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, StoredIndex, + VecIterator, Version, +}; + +const DCA_AMOUNT: Dollars = Dollars::mint(100.0); + +pub trait ComputeDCAStackViaLen { + fn compute_dca_stack_via_len( + &mut self, + max_from: DateIndex, + closes: &impl AnyIterableVec>, + len: usize, + exit: &Exit, + ) -> Result<()>; + + fn compute_dca_stack_via_from( + &mut self, + max_from: DateIndex, + closes: &impl AnyIterableVec>, + from: DateIndex, + exit: &Exit, + ) -> Result<()>; +} +impl ComputeDCAStackViaLen for EagerVec { + fn compute_dca_stack_via_len( + &mut self, + max_from: DateIndex, + closes: &impl AnyIterableVec>, + len: usize, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner_version() + closes.version(), + )?; + + let mut other_iter = closes.iter(); + let mut prev = None; + + let index = max_from.min(DateIndex::from(self.len())); + closes.iter_at(index).try_for_each(|(i, closes)| { + let price = *closes.into_owned(); + let i_usize = i.unwrap_to_usize(); + if prev.is_none() { + if i_usize == 0 { + prev.replace(Sats::ZERO); + } else { + prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1)); + } + } + + let mut stack = Sats::ZERO; + + if price != Dollars::ZERO { + stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price)); + + if i_usize >= len { + let prev_price = *other_iter.unwrap_get_inner_(i_usize - len); + if prev_price != Dollars::ZERO { + stack = stack + .checked_sub(Sats::from(Bitcoin::from(DCA_AMOUNT / prev_price))) + .unwrap(); + } + } + } + + prev.replace(stack); + + self.forced_push_at(i, stack, exit) + })?; + + self.safe_flush(exit)?; + + Ok(()) + } + + fn compute_dca_stack_via_from( + &mut self, + max_from: DateIndex, + closes: &impl AnyIterableVec>, + from: DateIndex, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner_version() + closes.version(), + )?; + + let mut prev = None; + + let index = max_from.min(DateIndex::from(self.len())); + closes.iter_at(index).try_for_each(|(i, closes)| { + let price = *closes.into_owned(); + let i_usize = i.unwrap_to_usize(); + if prev.is_none() { + if i_usize == 0 { + prev.replace(Sats::ZERO); + } else { + prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1)); + } + } + + let mut stack = Sats::ZERO; + + if price != Dollars::ZERO && i >= from { + stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price)); + } + + prev.replace(stack); + + self.forced_push_at(i, stack, exit) + })?; + + self.safe_flush(exit)?; + + Ok(()) + } +} + +pub trait ComputeDCAAveragePriceViaLen { + fn compute_dca_avg_price_via_len( + &mut self, + max_from: DateIndex, + stacks: &impl AnyIterableVec, + len: usize, + exit: &Exit, + ) -> Result<()>; + + fn compute_dca_avg_price_via_from( + &mut self, + max_from: DateIndex, + stacks: &impl AnyIterableVec, + from: DateIndex, + exit: &Exit, + ) -> Result<()>; +} +impl ComputeDCAAveragePriceViaLen for EagerVec { + fn compute_dca_avg_price_via_len( + &mut self, + max_from: DateIndex, + stacks: &impl AnyIterableVec, + len: usize, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ONE + self.inner_version() + stacks.version(), + )?; + + let index = max_from.min(DateIndex::from(self.len())); + + let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap(); + + stacks.iter_at(index).try_for_each(|(i, stack)| { + let stack = stack.into_owned(); + let mut avg_price = Dollars::from(f64::NAN); + if i > first_price_date { + avg_price = DCA_AMOUNT + * len + .min(i.unwrap_to_usize() + 1) + .min(i.checked_sub(first_price_date).unwrap().unwrap_to_usize() + 1) + / Bitcoin::from(stack); + } + self.forced_push_at(i, avg_price, exit) + })?; + + self.safe_flush(exit)?; + + Ok(()) + } + + fn compute_dca_avg_price_via_from( + &mut self, + max_from: DateIndex, + stacks: &impl AnyIterableVec, + from: DateIndex, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner_version() + stacks.version(), + )?; + + let index = max_from.min(DateIndex::from(self.len())); + + let from_usize = from.unwrap_to_usize(); + + stacks.iter_at(index).try_for_each(|(i, stack)| { + let stack = stack.into_owned(); + let mut avg_price = Dollars::from(f64::NAN); + if i >= from { + avg_price = + DCA_AMOUNT * (i.unwrap_to_usize() + 1 - from_usize) / Bitcoin::from(stack); + } + self.forced_push_at(i, avg_price, exit) + })?; + + self.safe_flush(exit)?; + + Ok(()) + } +} + +pub trait ComputeFromSats { + fn compute_from_sats( + &mut self, + max_from: I, + sats: &impl AnyIterableVec, + exit: &Exit, + ) -> Result<()>; +} +impl ComputeFromSats for EagerVec +where + I: StoredIndex, +{ + fn compute_from_sats( + &mut self, + max_from: I, + sats: &impl AnyIterableVec, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner_version() + sats.version(), + )?; + + let index = max_from.min(I::from(self.len())); + sats.iter_at(index).try_for_each(|(i, sats)| { + let (i, v) = (i, Bitcoin::from(sats.into_owned())); + self.forced_push_at(i, v, exit) + })?; + + self.safe_flush(exit)?; + + Ok(()) + } +} + +pub trait ComputeFromBitcoin { + fn compute_from_bitcoin( + &mut self, + max_from: I, + bitcoin: &impl AnyIterableVec, + price: &impl AnyIterableVec>, + exit: &Exit, + ) -> Result<()>; +} +impl ComputeFromBitcoin for EagerVec +where + I: StoredIndex, +{ + fn compute_from_bitcoin( + &mut self, + max_from: I, + bitcoin: &impl AnyIterableVec, + price: &impl AnyIterableVec>, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner_version() + bitcoin.version(), + )?; + + let mut price_iter = price.iter(); + let index = max_from.min(I::from(self.len())); + bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| { + let dollars = price_iter.unwrap_get_inner(i); + let (i, v) = (i, *dollars * bitcoin.into_owned()); + self.forced_push_at(i, v, exit) + })?; + + self.safe_flush(exit)?; + + Ok(()) + } +} + +pub trait ComputeDrawdown { + fn compute_drawdown( + &mut self, + max_from: I, + close: &impl AnyIterableVec>, + ath: &impl AnyIterableVec, + exit: &Exit, + ) -> Result<()>; +} +impl ComputeDrawdown for EagerVec +where + I: StoredIndex, +{ + fn compute_drawdown( + &mut self, + max_from: I, + close: &impl AnyIterableVec>, + ath: &impl AnyIterableVec, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner_version() + ath.version() + close.version(), + )?; + + let index = max_from.min(I::from(self.len())); + let mut close_iter = close.iter(); + ath.iter_at(index).try_for_each(|(i, ath)| { + let ath = ath.into_owned(); + if ath == Dollars::ZERO { + self.forced_push_at(i, StoredF32::default(), exit) + } else { + let close = *close_iter.unwrap_get_inner(i); + let drawdown = StoredF32::from((*ath - *close) / *ath * -100.0); + self.forced_push_at(i, drawdown, exit) + } + })?; + + self.safe_flush(exit)?; + + Ok(()) + } +} diff --git a/crates/brk_computer/src/transactions.rs b/crates/brk_computer/src/transactions.rs index 1c54e2dbe..d6b9aa615 100644 --- a/crates/brk_computer/src/transactions.rs +++ b/crates/brk_computer/src/transactions.rs @@ -1,14 +1,15 @@ -use std::sync::Arc; +use std::{path::Path, sync::Arc}; -use brk_core::{ - CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32, - StoredUsize, TxIndex, TxVersion, Version, Weight, -}; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; +use brk_structs::{ + CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredBool, + StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight, +}; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec, - ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, File, Format, StoredIndex, VecIterator, + AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec, + ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, StoredIndex, + VecIterator, }; use crate::grouped::{ @@ -16,89 +17,86 @@ use crate::grouped::{ ComputedVecsFromTxindex, Source, VecBuilderOptions, }; -use super::{Indexes, fetched, indexes}; +use super::{Indexes, indexes, price}; const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { + file: Arc, + // pub txindex_to_is_v1: LazyVec, // pub txindex_to_is_v2: LazyVec, // pub txindex_to_is_v3: LazyVec, pub indexes_to_coinbase: ComputedValueVecsFromHeight, - pub indexes_to_emptyoutput_count: ComputedVecsFromHeight, + pub indexes_to_emptyoutput_count: ComputedVecsFromHeight, pub indexes_to_fee: ComputedValueVecsFromTxindex, pub indexes_to_feerate: ComputedVecsFromTxindex, /// Value == 0 when Coinbase - pub txindex_to_input_value: ComputedVecFrom3< - TxIndex, - Sats, - TxIndex, - InputIndex, - TxIndex, - StoredUsize, - InputIndex, - Sats, - >, + pub txindex_to_input_value: + ComputedVecFrom3, // pub indexes_to_input_value: ComputedVecsFromTxindex, - pub indexes_to_opreturn_count: ComputedVecsFromHeight, + pub indexes_to_opreturn_count: ComputedVecsFromHeight, pub txindex_to_output_value: ComputedVecFrom3< TxIndex, Sats, TxIndex, OutputIndex, TxIndex, - StoredUsize, + StoredU64, OutputIndex, Sats, >, // pub indexes_to_output_value: ComputedVecsFromTxindex, - pub indexes_to_p2a_count: ComputedVecsFromHeight, - pub indexes_to_p2ms_count: ComputedVecsFromHeight, - pub indexes_to_p2pk33_count: ComputedVecsFromHeight, - pub indexes_to_p2pk65_count: ComputedVecsFromHeight, - pub indexes_to_p2pkh_count: ComputedVecsFromHeight, - pub indexes_to_p2sh_count: ComputedVecsFromHeight, - pub indexes_to_p2tr_count: ComputedVecsFromHeight, - pub indexes_to_p2wpkh_count: ComputedVecsFromHeight, - pub indexes_to_p2wsh_count: ComputedVecsFromHeight, + pub indexes_to_p2a_count: ComputedVecsFromHeight, + pub indexes_to_p2ms_count: ComputedVecsFromHeight, + pub indexes_to_p2pk33_count: ComputedVecsFromHeight, + pub indexes_to_p2pk65_count: ComputedVecsFromHeight, + pub indexes_to_p2pkh_count: ComputedVecsFromHeight, + pub indexes_to_p2sh_count: ComputedVecsFromHeight, + pub indexes_to_p2tr_count: ComputedVecsFromHeight, + pub indexes_to_p2wpkh_count: ComputedVecsFromHeight, + pub indexes_to_p2wsh_count: ComputedVecsFromHeight, pub indexes_to_subsidy: ComputedValueVecsFromHeight, pub indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight, - pub indexes_to_tx_count: ComputedVecsFromHeight, - pub indexes_to_tx_v1: ComputedVecsFromHeight, - pub indexes_to_tx_v2: ComputedVecsFromHeight, - pub indexes_to_tx_v3: ComputedVecsFromHeight, - pub indexes_to_tx_vsize: ComputedVecsFromTxindex, + pub indexes_to_tx_count: ComputedVecsFromHeight, + pub indexes_to_tx_v1: ComputedVecsFromHeight, + pub indexes_to_tx_v2: ComputedVecsFromHeight, + pub indexes_to_tx_v3: ComputedVecsFromHeight, + pub indexes_to_tx_vsize: ComputedVecsFromTxindex, pub indexes_to_tx_weight: ComputedVecsFromTxindex, - pub indexes_to_unknownoutput_count: ComputedVecsFromHeight, + pub indexes_to_unknownoutput_count: ComputedVecsFromHeight, pub inputindex_to_value: ComputedVecFrom2, - pub indexes_to_input_count: ComputedVecsFromTxindex, - pub txindex_to_is_coinbase: ComputedVecFrom2, - pub indexes_to_output_count: ComputedVecsFromTxindex, - pub txindex_to_vsize: ComputedVecFrom1, + pub indexes_to_input_count: ComputedVecsFromTxindex, + pub txindex_to_is_coinbase: + ComputedVecFrom2, + pub indexes_to_output_count: ComputedVecsFromTxindex, + pub txindex_to_vsize: ComputedVecFrom1, pub txindex_to_weight: ComputedVecFrom2, pub txindex_to_fee: ComputedVecFrom2, - pub txindex_to_feerate: ComputedVecFrom2, - pub indexes_to_exact_utxo_count: ComputedVecsFromHeight, + pub txindex_to_feerate: ComputedVecFrom2, + pub indexes_to_exact_utxo_count: ComputedVecsFromHeight, } impl Vecs { pub fn forced_import( - file: &Arc, + parent: &Path, version: Version, indexer: &Indexer, indexes: &indexes::Vecs, computation: Computation, format: Format, - fetched: Option<&fetched::Vecs>, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); + price: Option<&price::Vecs>, + ) -> Result { + let file = Arc::new(File::open(&parent.join("transactions"))?); + + let compute_dollars = price.is_some(); let inputindex_to_value = ComputedVec::forced_import_or_init_from_2( computation, - file, + &file, "value", version + VERSION + Version::ZERO, format, @@ -125,7 +123,7 @@ impl Vecs { let txindex_to_weight = ComputedVec::forced_import_or_init_from_2( computation, - file, + &file, "weight", version + VERSION + Version::ZERO, format, @@ -153,7 +151,7 @@ impl Vecs { let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1( computation, - file, + &file, "vsize", version + VERSION + Version::ZERO, format, @@ -161,7 +159,7 @@ impl Vecs { |index: TxIndex, iter| { let index = index.unwrap_to_usize(); iter.next_at(index).map(|(_, weight)| { - StoredUsize::from( + StoredU64::from( bitcoin::Weight::from(weight.into_owned()).to_vbytes_ceil() as usize ) }) @@ -170,7 +168,7 @@ impl Vecs { let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2( computation, - file, + &file, "is_coinbase", version + VERSION + Version::ZERO, format, @@ -186,15 +184,14 @@ impl Vecs { .unwrap() .1 .into_owned(); - - index == txindex + StoredBool::from(index == txindex) }) }, )?; let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3( computation, - file, + &file, "input_value", version + VERSION + Version::ZERO, format, @@ -215,7 +212,7 @@ impl Vecs { .unwrap() .1 .into_owned(); - let range = first_index..first_index + count; + let range = first_index..first_index + count as usize; range.into_iter().fold(Sats::ZERO, |total, inputindex| { total + inputindex_to_value_iter @@ -244,7 +241,7 @@ impl Vecs { let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3( computation, - file, + &file, "output_value", version + VERSION + Version::ZERO, format, @@ -265,7 +262,7 @@ impl Vecs { .unwrap() .1 .into_owned(); - let range = first_index..first_index + count; + let range = first_index..first_index + count as usize; range.into_iter().fold(Sats::ZERO, |total, outputindex| { total + outputindex_to_value_iter @@ -294,7 +291,7 @@ impl Vecs { let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2( computation, - file, + &file, "fee", version + VERSION + Version::ZERO, format, @@ -317,7 +314,7 @@ impl Vecs { let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2( computation, - file, + &file, "feerate", version + VERSION + Version::ZERO, format, @@ -337,7 +334,7 @@ impl Vecs { Ok(Self { indexes_to_tx_count: ComputedVecsFromHeight::forced_import( - file, + &file, "tx_count", Source::Compute, version + VERSION + Version::ZERO, @@ -352,7 +349,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_input_count: ComputedVecsFromTxindex::forced_import( - file, + &file, "input_count", Source::None, version + VERSION + Version::ZERO, @@ -367,7 +364,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_output_count: ComputedVecsFromTxindex::forced_import( - file, + &file, "output_count", Source::None, version + VERSION + Version::ZERO, @@ -382,7 +379,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( - file, + &file, "tx_v1", Source::Compute, version + VERSION + Version::ZERO, @@ -392,7 +389,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v2: ComputedVecsFromHeight::forced_import( - file, + &file, "tx_v2", Source::Compute, version + VERSION + Version::ZERO, @@ -402,7 +399,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v3: ComputedVecsFromHeight::forced_import( - file, + &file, "tx_v3", Source::Compute, version + VERSION + Version::ZERO, @@ -412,14 +409,14 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_fee: ComputedValueVecsFromTxindex::forced_import( - file, + &file, "fee", indexes, Source::Vec(txindex_to_fee.boxed_clone()), version + VERSION + Version::ZERO, computation, format, - fetched, + price, VecBuilderOptions::default() .add_sum() .add_cumulative() @@ -428,7 +425,7 @@ impl Vecs { .add_average(), )?, indexes_to_feerate: ComputedVecsFromTxindex::forced_import( - file, + &file, "feerate", Source::None, version + VERSION + Version::ZERO, @@ -441,7 +438,7 @@ impl Vecs { .add_average(), )?, indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import( - file, + &file, "tx_vsize", Source::None, version + VERSION + Version::ZERO, @@ -454,7 +451,7 @@ impl Vecs { .add_average(), )?, indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import( - file, + &file, "tx_weight", Source::None, version + VERSION + Version::ZERO, @@ -467,7 +464,7 @@ impl Vecs { .add_average(), )?, indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( - file, + &file, "subsidy", Source::Compute, version + VERSION + Version::ZERO, @@ -483,7 +480,7 @@ impl Vecs { indexes, )?, indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import( - file, + &file, "coinbase", Source::Compute, version + VERSION + Version::ZERO, @@ -499,7 +496,7 @@ impl Vecs { indexes, )?, indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import( - file, + &file, "unclaimed_rewards", Source::Compute, version + VERSION + Version::ZERO, @@ -510,7 +507,7 @@ impl Vecs { indexes, )?, indexes_to_p2a_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2a_count", Source::Compute, version + VERSION + Version::ZERO, @@ -525,7 +522,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2ms_count", Source::Compute, version + VERSION + Version::ZERO, @@ -540,7 +537,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pk33_count", Source::Compute, version + VERSION + Version::ZERO, @@ -555,7 +552,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pk65_count", Source::Compute, version + VERSION + Version::ZERO, @@ -570,7 +567,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2pkh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -585,7 +582,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2sh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -600,7 +597,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2tr_count", Source::Compute, version + VERSION + Version::ZERO, @@ -615,7 +612,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2wpkh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -630,7 +627,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import( - file, + &file, "p2wsh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -645,7 +642,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import( - file, + &file, "opreturn_count", Source::Compute, version + VERSION + Version::ZERO, @@ -660,7 +657,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import( - file, + &file, "unknownoutput_count", Source::Compute, version + VERSION + Version::ZERO, @@ -675,7 +672,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import( - file, + &file, "emptyoutput_count", Source::Compute, version + VERSION + Version::ZERO, @@ -690,7 +687,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import( - file, + &file, "exact_utxo_count", Source::Compute, version + VERSION + Version::ZERO, @@ -709,6 +706,8 @@ impl Vecs { txindex_to_feerate, txindex_to_vsize, txindex_to_weight, + + file, }) } @@ -717,9 +716,9 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, starting_indexes: &Indexes, - fetched: Option<&fetched::Vecs>, + price: Option<&price::Vecs>, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> Result<()> { self.indexes_to_tx_count.compute_all( indexer, indexes, @@ -731,7 +730,8 @@ impl Vecs { &indexer.vecs.height_to_first_txindex, &indexer.vecs.txindex_to_txid, exit, - ) + )?; + Ok(()) }, )?; @@ -752,7 +752,7 @@ impl Vecs { )?; let compute_indexes_to_tx_vany = - |indexes_to_tx_vany: &mut ComputedVecsFromHeight, txversion| { + |indexes_to_tx_vany: &mut ComputedVecsFromHeight, txversion| { let mut txindex_to_txversion_iter = indexer.vecs.txindex_to_txversion.iter(); indexes_to_tx_vany.compute_all( indexer, @@ -769,7 +769,8 @@ impl Vecs { v == txversion }, exit, - ) + )?; + Ok(()) }, ) }; @@ -863,7 +864,7 @@ impl Vecs { starting_indexes, exit, Some(&self.txindex_to_fee), - fetched, + price, )?; self.indexes_to_feerate.compute_rest( @@ -893,7 +894,7 @@ impl Vecs { self.indexes_to_coinbase.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |vec, indexer, _, starting_indexes, exit| { @@ -910,23 +911,23 @@ impl Vecs { .unwrap_to_usize(); let output_count = txindex_to_output_count_iter.unwrap_get_inner(txindex); let mut sats = Sats::ZERO; - (first_outputindex..first_outputindex + *output_count).for_each( - |outputindex| { + (first_outputindex..first_outputindex + usize::from(output_count)) + .for_each(|outputindex| { sats += outputindex_to_value_iter .unwrap_get_inner(OutputIndex::from(outputindex)); - }, - ); + }); (height, sats) }, exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_subsidy.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { @@ -940,14 +941,15 @@ impl Vecs { (height, coinbase.checked_sub(fees).unwrap()) }, exit, - ) + )?; + Ok(()) }, )?; self.indexes_to_unclaimed_rewards.compute_all( indexer, indexes, - fetched, + price, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { @@ -961,7 +963,8 @@ impl Vecs { (height, expected.checked_sub(subsidy).unwrap()) }, exit, - ) + )?; + Ok(()) }, )?; @@ -976,7 +979,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2aaddressindex, &indexer.vecs.p2aaddressindex_to_p2abytes, exit, - ) + )?; + Ok(()) }, )?; @@ -991,7 +995,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2msoutputindex, &indexer.vecs.p2msoutputindex_to_txindex, exit, - ) + )?; + Ok(()) }, )?; @@ -1006,7 +1011,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2pk33addressindex, &indexer.vecs.p2pk33addressindex_to_p2pk33bytes, exit, - ) + )?; + Ok(()) }, )?; @@ -1021,7 +1027,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2pk65addressindex, &indexer.vecs.p2pk65addressindex_to_p2pk65bytes, exit, - ) + )?; + Ok(()) }, )?; @@ -1036,7 +1043,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2pkhaddressindex, &indexer.vecs.p2pkhaddressindex_to_p2pkhbytes, exit, - ) + )?; + Ok(()) }, )?; @@ -1051,7 +1059,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2shaddressindex, &indexer.vecs.p2shaddressindex_to_p2shbytes, exit, - ) + )?; + Ok(()) }, )?; @@ -1066,7 +1075,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2traddressindex, &indexer.vecs.p2traddressindex_to_p2trbytes, exit, - ) + )?; + Ok(()) }, )?; @@ -1081,7 +1091,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2wpkhaddressindex, &indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes, exit, - ) + )?; + Ok(()) }, )?; @@ -1096,7 +1107,8 @@ impl Vecs { &indexer.vecs.height_to_first_p2wshaddressindex, &indexer.vecs.p2wshaddressindex_to_p2wshbytes, exit, - ) + )?; + Ok(()) }, )?; @@ -1111,7 +1123,8 @@ impl Vecs { &indexer.vecs.height_to_first_opreturnindex, &indexer.vecs.opreturnindex_to_txindex, exit, - ) + )?; + Ok(()) }, )?; @@ -1126,7 +1139,8 @@ impl Vecs { &indexer.vecs.height_to_first_unknownoutputindex, &indexer.vecs.unknownoutputindex_to_txindex, exit, - ) + )?; + Ok(()) }, )?; @@ -1141,7 +1155,8 @@ impl Vecs { &indexer.vecs.height_to_first_emptyoutputindex, &indexer.vecs.emptyoutputindex_to_txindex, exit, - ) + )?; + Ok(()) }, )?; @@ -1167,7 +1182,7 @@ impl Vecs { |(h, output_count, ..)| { let input_count = input_count_iter.unwrap_get_inner(h); let opreturn_count = opreturn_count_iter.unwrap_get_inner(h); - let block_count = usize::from(h + 1_usize); + let block_count = u64::from(h + 1_usize); // -1 > genesis output is unspendable let mut utxo_count = *output_count - (*input_count - block_count) - *opreturn_count - 1; @@ -1190,13 +1205,16 @@ impl Vecs { utxo_count -= 1; } - (h, StoredUsize::from(utxo_count)) + (h, StoredU64::from(utxo_count)) }, exit, - ) + )?; + Ok(()) }, )?; + self.file.flush()?; + self.file.punch_holes()?; Ok(()) } diff --git a/crates/brk_core/src/structs/_addressindex.rs b/crates/brk_core/src/structs/_addressindex.rs deleted file mode 100644 index bde3019d4..000000000 --- a/crates/brk_core/src/structs/_addressindex.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::ops::Add; - -use byteview::ByteView; -use serde::Serialize; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use crate::Error; - -#[derive( - Debug, - PartialEq, - Eq, - PartialOrd, - Ord, - Clone, - Copy, - Default, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Serialize, -)] -pub struct AddressIndex(u32); - -impl AddressIndex { - pub const BYTES: usize = size_of::(); - - pub fn increment(&mut self) { - self.0 += 1; - } - - pub fn incremented(self) -> Self { - Self(self.0 + 1) - } -} - -impl From for AddressIndex { - fn from(value: u32) -> Self { - Self(value) - } -} - -impl From for AddressIndex { - fn from(value: u64) -> Self { - Self(value as u32) - } -} -impl From for u64 { - fn from(value: AddressIndex) -> Self { - value.0 as u64 - } -} - -impl From for AddressIndex { - fn from(value: usize) -> Self { - Self(value as u32) - } -} -impl From for usize { - fn from(value: AddressIndex) -> Self { - value.0 as usize - } -} - -impl From for AddressIndex { - fn from(value: ByteView) -> Self { - Ok(Self::read_from_bytes(&value)?) - } -} -impl From for ByteView { - fn from(value: AddressIndex) -> Self { - Self::new(value.as_bytes()) - } -} - -impl Add for AddressIndex { - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - Self(self.0 + rhs as u32) - } -} - -impl Add for AddressIndex { - type Output = Self; - fn add(self, rhs: AddressIndex) -> Self::Output { - Self(self.0 + rhs.0) - } -} diff --git a/crates/brk_core/src/structs/_addressindexoutputindex.rs b/crates/brk_core/src/structs/_addressindexoutputindex.rs deleted file mode 100644 index f37bb50ec..000000000 --- a/crates/brk_core/src/structs/_addressindexoutputindex.rs +++ /dev/null @@ -1,27 +0,0 @@ -use byteview::ByteView; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use crate::Error; - -use super::{AddressIndex, Outputindex}; - -#[derive( - Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes, -)] -#[repr(C)] -pub struct AddressIndexOutputIndex { - addressindex: AddressIndex, - _padding: u32, - outputindex: Outputindex, -} - -impl From for AddressIndexOutputIndex { - fn from(value: ByteView) -> Self { - Ok(Self::read_from_bytes(&value)?) - } -} -impl From for ByteView { - fn from(value: AddressIndexOutputIndex) -> Self { - Self::new(value.as_bytes()) - } -} diff --git a/crates/brk_core/src/structs/_addresstype.rs b/crates/brk_core/src/structs/_addresstype.rs deleted file mode 100644 index fb1705148..000000000 --- a/crates/brk_core/src/structs/_addresstype.rs +++ /dev/null @@ -1,48 +0,0 @@ -use serde::Serialize; -use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; - -use super::OutputType; - -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - TryFromBytes, - Immutable, - IntoBytes, - KnownLayout, - Serialize, -)] -#[repr(u8)] -pub enum AddressType { - P2PK65, - P2PK33, - P2PKH, - P2SH, - P2WPKH, - P2WSH, - P2TR, - P2A, -} - -impl From for AddressType { - fn from(value: OutputType) -> Self { - match value { - OutputType::P2A => Self::P2A, - OutputType::P2PK33 => Self::P2PK33, - OutputType::P2PK65 => Self::P2PK65, - OutputType::P2PKH => Self::P2PKH, - OutputType::P2SH => Self::P2SH, - OutputType::P2TR => Self::P2TR, - OutputType::P2WPKH => Self::P2WPKH, - OutputType::P2WSH => Self::P2WSH, - OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => { - unreachable!() - } - } - } -} diff --git a/crates/brk_core/src/structs/outputtype.rs b/crates/brk_core/src/structs/outputtype.rs deleted file mode 100644 index 4f1eab597..000000000 --- a/crates/brk_core/src/structs/outputtype.rs +++ /dev/null @@ -1,135 +0,0 @@ -use bitcoin::{ScriptBuf, opcodes::all::OP_PUSHBYTES_2}; -use serde::Serialize; -use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; - -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - TryFromBytes, - Immutable, - IntoBytes, - KnownLayout, - Serialize, -)] -#[repr(u8)] -pub enum OutputType { - P2PK65, - P2PK33, - P2PKH, - P2MS, - P2SH, - OpReturn, - P2WPKH, - P2WSH, - P2TR, - P2A, - Empty = 254, - Unknown = 255, -} - -impl OutputType { - pub fn is_spendable(&self) -> bool { - match self { - Self::P2PK65 => true, - Self::P2PK33 => true, - Self::P2PKH => true, - Self::P2MS => true, - Self::P2SH => true, - Self::OpReturn => false, - Self::P2WPKH => true, - Self::P2WSH => true, - Self::P2TR => true, - Self::P2A => true, - Self::Empty => true, - Self::Unknown => true, - } - } - - pub fn is_address(&self) -> bool { - match self { - Self::P2PK65 => true, - Self::P2PK33 => true, - Self::P2PKH => true, - Self::P2MS => false, - Self::P2SH => true, - Self::OpReturn => false, - Self::P2WPKH => true, - Self::P2WSH => true, - Self::P2TR => true, - Self::P2A => true, - Self::Empty => false, - Self::Unknown => false, - } - } - - pub fn is_not_address(&self) -> bool { - !self.is_address() - } - - pub fn is_unspendable(&self) -> bool { - !self.is_spendable() - } - - pub fn as_vec() -> Vec { - vec![ - Self::P2PK65, - Self::P2PK33, - Self::P2PKH, - Self::P2MS, - Self::P2SH, - Self::OpReturn, - Self::P2WPKH, - Self::P2WSH, - Self::P2TR, - Self::P2A, - Self::Empty, - Self::Unknown, - ] - } -} - -impl From<&ScriptBuf> for OutputType { - fn from(script: &ScriptBuf) -> Self { - if script.is_p2pk() { - let bytes = script.as_bytes(); - - match bytes.len() { - 67 => Self::P2PK65, - 35 => Self::P2PK33, - _ => { - dbg!(bytes); - unreachable!() - } - } - } else if script.is_p2pkh() { - Self::P2PKH - } else if script.is_multisig() { - Self::P2MS - } else if script.is_p2sh() { - Self::P2SH - } else if script.is_op_return() { - Self::OpReturn - } else if script.is_p2wpkh() { - Self::P2WPKH - } else if script.is_p2wsh() { - Self::P2WSH - } else if script.is_p2tr() { - Self::P2TR - } else if script.witness_version() == Some(bitcoin::WitnessVersion::V1) - && script.len() == 4 - && script.as_bytes()[1] == OP_PUSHBYTES_2.to_u8() - && script.as_bytes()[2..4] == [78, 115] - { - Self::P2A - } else if script.is_empty() { - Self::Empty - } else { - Self::Unknown - } - } -} diff --git a/crates/brk_core/src/structs/stored_u64.rs b/crates/brk_core/src/structs/stored_u64.rs deleted file mode 100644 index 82a8908dc..000000000 --- a/crates/brk_core/src/structs/stored_u64.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::ops::{Add, AddAssign, Div}; - -use derive_deref::Deref; -use serde::Serialize; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use crate::{CheckedSub, Printable}; - -use super::{InputIndex, OutputIndex, TxIndex}; - -#[derive( - Debug, - Deref, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Serialize, -)] -pub struct StoredU64(u64); - -impl StoredU64 { - pub const ZERO: Self = Self(0); - - pub fn new(counter: u64) -> Self { - Self(counter) - } -} - -impl From for StoredU64 { - fn from(value: u64) -> Self { - Self(value) - } -} - -impl From for StoredU64 { - fn from(value: usize) -> Self { - Self(value as u64) - } -} - -impl CheckedSub for StoredU64 { - fn checked_sub(self, rhs: Self) -> Option { - self.0.checked_sub(rhs.0).map(Self) - } -} - -impl Div for StoredU64 { - type Output = Self; - fn div(self, rhs: usize) -> Self::Output { - Self(self.0 / rhs as u64) - } -} - -impl Add for StoredU64 { - type Output = Self; - fn add(self, rhs: Self) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -impl AddAssign for StoredU64 { - fn add_assign(&mut self, rhs: Self) { - *self = *self + rhs - } -} - -impl From for StoredU64 { - fn from(value: f64) -> Self { - if value < 0.0 || value > u32::MAX as f64 { - panic!() - } - Self(value as u64) - } -} - -impl From for f64 { - fn from(value: StoredU64) -> Self { - value.0 as f64 - } -} - -impl From for StoredU64 { - fn from(value: TxIndex) -> Self { - Self(*value as u64) - } -} - -impl From for StoredU64 { - fn from(value: InputIndex) -> Self { - Self(*value) - } -} - -impl From for StoredU64 { - fn from(value: OutputIndex) -> Self { - Self(*value) - } -} - -impl Printable for StoredU64 { - fn to_string() -> &'static str { - "u64" - } - - fn to_possible_strings() -> &'static [&'static str] { - &["u64"] - } -} diff --git a/crates/brk_core/src/traits/mod.rs b/crates/brk_core/src/traits/mod.rs deleted file mode 100644 index 8e724b8e7..000000000 --- a/crates/brk_core/src/traits/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod from_coarser; -mod printable; - -pub use from_coarser::*; -pub use printable::*; diff --git a/crates/brk_core/src/traits/printable.rs b/crates/brk_core/src/traits/printable.rs deleted file mode 100644 index 6689b9665..000000000 --- a/crates/brk_core/src/traits/printable.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub trait Printable { - fn to_string() -> &'static str; - fn to_possible_strings() -> &'static [&'static str]; -} diff --git a/crates/brk_core/src/utils/bytes.rs b/crates/brk_core/src/utils/bytes.rs deleted file mode 100644 index 476058e49..000000000 --- a/crates/brk_core/src/utils/bytes.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::{Error, Result}; - -#[allow(clippy::result_unit_err)] -pub fn copy_first_4bytes(slice: &[u8]) -> Result<[u8; 4]> { - let mut buf: [u8; 4] = [0; 4]; - let buf_len = buf.len(); - if slice.len() < buf_len { - return Err(Error::Str("Buffer is too small to convert to 8 bytes")); - } - slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { - buf[i] = *r; - }); - Ok(buf) -} - -#[allow(clippy::result_unit_err)] -pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> { - let mut buf: [u8; 8] = [0; 8]; - let buf_len = buf.len(); - if slice.len() < buf_len { - return Err(Error::Str("Buffer is too small to convert to 8 bytes")); - } - slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { - buf[i] = *r; - }); - Ok(buf) -} diff --git a/crates/brk_core/src/utils/mod.rs b/crates/brk_core/src/utils/mod.rs deleted file mode 100644 index b410029b1..000000000 --- a/crates/brk_core/src/utils/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod bytes; -mod checked_sub; -mod paths; -mod pause; -mod rlimit; -mod serde; - -pub use bytes::*; -pub use checked_sub::*; -pub use paths::*; -pub use pause::*; -pub use rlimit::*; -pub use serde::*; diff --git a/crates/brk_core/src/utils/pause.rs b/crates/brk_core/src/utils/pause.rs deleted file mode 100644 index 4be9ea29b..000000000 --- a/crates/brk_core/src/utils/pause.rs +++ /dev/null @@ -1,9 +0,0 @@ -use log::info; - -pub fn pause() { - info!("Press enter to continue..."); - let mut buffer = String::new(); - std::io::stdin() - .read_line(&mut buffer) - .expect("Failed to read line"); -} diff --git a/crates/brk_core/src/utils/rlimit.rs b/crates/brk_core/src/utils/rlimit.rs deleted file mode 100644 index f725c7b96..000000000 --- a/crates/brk_core/src/utils/rlimit.rs +++ /dev/null @@ -1,15 +0,0 @@ -use std::io; - -use rlimit::{Resource, getrlimit}; - -pub fn setrlimit() -> io::Result<()> { - let no_file_limit = getrlimit(Resource::NOFILE)?; - - rlimit::setrlimit( - Resource::NOFILE, - no_file_limit.0.max(10_000), - no_file_limit.1, - )?; - - Ok(()) -} diff --git a/crates/brk_core/src/utils/serde.rs b/crates/brk_core/src/utils/serde.rs deleted file mode 100644 index b3ae164a6..000000000 --- a/crates/brk_core/src/utils/serde.rs +++ /dev/null @@ -1,12 +0,0 @@ -use serde::{Deserialize, Deserializer}; - -pub fn default_on_error<'de, D, T>(deserializer: D) -> Result -where - D: Deserializer<'de>, - T: Deserialize<'de> + Default, -{ - match T::deserialize(deserializer) { - Ok(v) => Ok(v), - Err(_) => Ok(T::default()), - } -} diff --git a/crates/brk_error/Cargo.toml b/crates/brk_error/Cargo.toml new file mode 100644 index 000000000..c6324f200 --- /dev/null +++ b/crates/brk_error/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "brk_error" +description = "Errors used throughout the Bitcoin Research Kit" +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +brk_vecs = { workspace = true } +bitcoincore-rpc = { workspace = true } +fjall = { workspace = true } +jiff = { workspace = true } +minreq = { workspace = true } +zerocopy = { workspace = true } diff --git a/crates/brk_core/README.md b/crates/brk_error/README.md similarity index 56% rename from crates/brk_core/README.md rename to crates/brk_error/README.md index 91c2ea9bf..4fe182961 100644 --- a/crates/brk_core/README.md +++ b/crates/brk_error/README.md @@ -7,15 +7,15 @@ License - - Version + + Version - - Documentation + + Documentation - Size - - Dependency status + Size + + Dependency status Discord @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A list of structs that are used throughout the project as units, think of `Date`, `Height`, `Sats`, `Txindex` or anything that can be either a key and/or a value of a dataset. diff --git a/crates/brk_error/src/lib.rs b/crates/brk_error/src/lib.rs new file mode 100644 index 000000000..df167ddd7 --- /dev/null +++ b/crates/brk_error/src/lib.rs @@ -0,0 +1,105 @@ +use std::{ + fmt::{self, Debug}, + io, result, time, +}; + +pub type Result = result::Result; + +#[derive(Debug)] +pub enum Error { + IO(io::Error), + BitcoinRPC(bitcoincore_rpc::Error), + Jiff(jiff::Error), + Fjall(fjall::Error), + Minreq(minreq::Error), + SystemTimeError(time::SystemTimeError), + ZeroCopyError, + Vecs(brk_vecs::Error), + + WrongLength, + WrongAddressType, + UnindexableDate, + Str(&'static str), + String(String), +} + +impl From for Error { + fn from(value: time::SystemTimeError) -> Self { + Self::SystemTimeError(value) + } +} + +impl From for Error { + fn from(value: io::Error) -> Self { + Self::IO(value) + } +} + +impl From for Error { + fn from(value: bitcoincore_rpc::Error) -> Self { + Self::BitcoinRPC(value) + } +} + +impl From for Error { + fn from(value: minreq::Error) -> Self { + Self::Minreq(value) + } +} + +impl From for Error { + fn from(value: brk_vecs::Error) -> Self { + Self::Vecs(value) + } +} + +impl From for Error { + fn from(value: jiff::Error) -> Self { + Self::Jiff(value) + } +} + +impl From for Error { + fn from(value: fjall::Error) -> Self { + Self::Fjall(value) + } +} + +impl From> for Error { + fn from(_: zerocopy::error::ConvertError) -> Self { + Self::ZeroCopyError + } +} + +impl From> for Error { + fn from(_: zerocopy::error::SizeError) -> Self { + Self::ZeroCopyError + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::IO(error) => Debug::fmt(&error, f), + Error::Minreq(error) => Debug::fmt(&error, f), + Error::Vecs(error) => Debug::fmt(&error, f), + Error::BitcoinRPC(error) => Debug::fmt(&error, f), + Error::SystemTimeError(error) => Debug::fmt(&error, f), + Error::Jiff(error) => Debug::fmt(&error, f), + Error::Fjall(error) => Debug::fmt(&error, f), + Error::ZeroCopyError => write!(f, "ZeroCopy error"), + + Error::WrongLength => write!(f, "Wrong length"), + Error::WrongAddressType => write!(f, "Wrong address type"), + Error::UnindexableDate => write!( + f, + "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater" + ), + + Error::Str(s) => write!(f, "{s}"), + Error::String(s) => write!(f, "{s}"), + } + } +} + +impl std::error::Error for Error {} diff --git a/crates/brk_exit/Cargo.toml b/crates/brk_exit/Cargo.toml deleted file mode 100644 index 58677bac4..000000000 --- a/crates/brk_exit/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "brk_exit" -description = "An exit blocker built on top of ctrlc" -version.workspace = true -edition.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true - -[dependencies] -brk_logger = { workspace = true } -ctrlc = { version = "3.4.7", features = ["termination"] } -log = { workspace = true } -parking_lot = { workspace = true } diff --git a/crates/brk_exit/README.md b/crates/brk_exit/README.md deleted file mode 100644 index 1ef11c665..000000000 --- a/crates/brk_exit/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# BRK Exit - -

- - GitHub Repo stars - - - License - - - Version - - - Documentation - - Size - - Dependency status - - - Discord - - - Nostr - - - Bluesky - - - X - -

- -A simple crate that stops the program from exitting when blocking is activated until it is released. The purpose of that is to prevent exitting when a program is in the middle of saving data and thus prevent partial writes. - -It's built on top of [ctrlc](https://crates.io/crates/ctrlc) which handles Ctrl + C (SIGINT), stopping the program using the `kill` command (SIGTERM) and closing the terminal (SIGHUP) but it doesn't support force kills (`kill -9`). diff --git a/crates/brk_exit/examples/main.rs b/crates/brk_exit/examples/main.rs deleted file mode 100644 index de192af1e..000000000 --- a/crates/brk_exit/examples/main.rs +++ /dev/null @@ -1,28 +0,0 @@ -use std::{path::Path, thread::sleep, time::Duration}; - -use brk_exit::Exit; -use log::info; - -fn main() { - let exit = Exit::new(); - - brk_logger::init(Some(Path::new(".log"))); - - let lock = exit.lock(); - - let mut i = 0; - while i < 21 { - info!("i = {i}"); - sleep(Duration::from_secs(1)); - i += 1; - } - - drop(lock); - - let mut j = 0; - while j < 10 { - info!("j = {j}"); - sleep(Duration::from_secs(1)); - j += 1; - } -} diff --git a/crates/brk_fetcher/Cargo.toml b/crates/brk_fetcher/Cargo.toml index ec5b1fec5..10bc4ad8f 100644 --- a/crates/brk_fetcher/Cargo.toml +++ b/crates/brk_fetcher/Cargo.toml @@ -8,9 +8,9 @@ homepage.workspace = true repository.workspace = true [dependencies] -brk_core = { workspace = true } +brk_error = { workspace = true } brk_logger = { workspace = true } -color-eyre = { workspace = true } +brk_structs = { workspace = true } log = { workspace = true } minreq = { workspace = true } serde_json = { workspace = true } diff --git a/crates/brk_fetcher/README.md b/crates/brk_fetcher/README.md index 722b03c35..c86905969 100644 --- a/crates/brk_fetcher/README.md +++ b/crates/brk_fetcher/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A crate that can fetch the Bitcoin price, either by date or height, from Binance, Kraken and the main instance of BRK. diff --git a/crates/brk_fetcher/examples/main.rs b/crates/brk_fetcher/examples/main.rs index 125fdb0bd..f1eb5ef26 100644 --- a/crates/brk_fetcher/examples/main.rs +++ b/crates/brk_fetcher/examples/main.rs @@ -1,9 +1,8 @@ -use brk_core::{Date, Height}; +use brk_error::Result; use brk_fetcher::{BRK, Binance, Fetcher, Kraken}; +use brk_structs::{Date, Height}; -fn main() -> color_eyre::Result<()> { - color_eyre::install()?; - +fn main() -> Result<()> { brk_logger::init(None); let mut brk = BRK::default(); diff --git a/crates/brk_fetcher/src/binance.rs b/crates/brk_fetcher/src/binance.rs index f4bf9dcaf..e7a0df0ed 100644 --- a/crates/brk_fetcher/src/binance.rs +++ b/crates/brk_fetcher/src/binance.rs @@ -6,8 +6,8 @@ use std::{ str::FromStr, }; -use brk_core::{Cents, OHLCCents, Timestamp}; -use color_eyre::eyre::{ContextCompat, eyre}; +use brk_error::{Error, Result}; +use brk_structs::{Cents, OHLCCents, Timestamp}; use log::info; use serde_json::Value; @@ -35,7 +35,7 @@ impl Binance { &mut self, timestamp: Timestamp, previous_timestamp: Option, - ) -> color_eyre::Result { + ) -> Result { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { @@ -65,7 +65,7 @@ impl Binance { ) } - pub fn fetch_1mn() -> color_eyre::Result> { + pub fn fetch_1mn() -> Result> { info!("Fetching 1mn prices from Binance..."); retry( @@ -81,7 +81,7 @@ impl Binance { ) } - pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { + pub fn get_from_1d(&mut self, date: &Date) -> Result { if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date { self._1d.replace(Self::fetch_1d()?); } @@ -91,10 +91,10 @@ impl Binance { .unwrap() .get(date) .cloned() - .ok_or(color_eyre::eyre::Error::msg("Couldn't find date")) + .ok_or(Error::Str("Couldn't find date")) } - pub fn fetch_1d() -> color_eyre::Result> { + pub fn fetch_1d() -> Result> { info!("Fetching daily prices from Binance..."); retry( @@ -104,9 +104,9 @@ impl Binance { ) } - fn read_har(&self) -> color_eyre::Result> { + fn read_har(&self) -> Result> { if self.path.is_none() { - return Err(eyre!("Path missing")); + return Err(Error::Str("Path missing")); } info!("Reading Binance har file..."); @@ -120,7 +120,7 @@ impl Binance { let file = if let Ok(file) = File::open(path_binance_har) { file } else { - return Err(eyre!("Missing binance file")); + return Err(Error::Str("Missing binance file")); }; let reader = BufReader::new(file); @@ -132,13 +132,13 @@ impl Binance { }; json.get("log") - .context("Expect object to have log attribute")? + .ok_or(Error::Str("Expect object to have log attribute"))? .as_object() - .context("Expect to be an object")? + .ok_or(Error::Str("Expect to be an object"))? .get("entries") - .context("Expect object to have entries")? + .ok_or(Error::Str("Expect object to have entries"))? .as_array() - .context("Expect to be an array")? + .ok_or(Error::Str("Expect to be an array"))? .iter() .filter(|entry| { entry @@ -181,30 +181,28 @@ impl Binance { }) } - fn json_to_timestamp_to_ohlc( - json: &Value, - ) -> color_eyre::Result> { + fn json_to_timestamp_to_ohlc(json: &Value) -> Result> { Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc) } - fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { + fn json_to_date_to_ohlc(json: &Value) -> Result> { Self::json_to_btree(json, Self::array_to_date_and_ohlc) } - fn json_to_btree(json: &Value, fun: F) -> color_eyre::Result> + fn json_to_btree(json: &Value, fun: F) -> Result> where - F: Fn(&Value) -> color_eyre::Result<(K, V)>, + F: Fn(&Value) -> Result<(K, V)>, K: Ord, { json.as_array() - .context("Expect to be an array")? + .ok_or(Error::Str("Expect to be an array"))? .iter() .map(fun) .collect::, _>>() } - fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> { - let array = array.as_array().context("Expect to be array")?; + fn array_to_timestamp_and_ohlc(array: &Value) -> Result<(Timestamp, OHLCCents)> { + let array = array.as_array().ok_or(Error::Str("Expect to be array"))?; let timestamp = Timestamp::from((array.first().unwrap().as_u64().unwrap() / 1_000) as u32); @@ -231,7 +229,7 @@ impl Binance { )) } - fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> { + fn array_to_date_and_ohlc(array: &Value) -> Result<(Date, OHLCCents)> { Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc)) } diff --git a/crates/brk_fetcher/src/brk.rs b/crates/brk_fetcher/src/brk.rs index 8b31eb542..30f573730 100644 --- a/crates/brk_fetcher/src/brk.rs +++ b/crates/brk_fetcher/src/brk.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; -use brk_core::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents}; -use color_eyre::eyre::{ContextCompat, eyre}; +use brk_error::{Error, Result}; +use brk_structs::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents}; use log::info; use serde_json::Value; @@ -19,7 +19,7 @@ const RETRIES: usize = 10; const CHUNK_SIZE: usize = 10_000; impl BRK { - pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result { + pub fn get_from_height(&mut self, height: Height) -> Result { let key = height.checked_sub(height % CHUNK_SIZE).unwrap(); #[allow(clippy::map_entry)] @@ -39,10 +39,10 @@ impl BRK { .unwrap() .get(usize::from(height.checked_sub(key).unwrap())) .cloned() - .ok_or(eyre!("Couldn't find height in BRK")) + .ok_or(Error::Str("Couldn't find height in BRK")) } - fn fetch_height_prices(height: Height) -> color_eyre::Result> { + fn fetch_height_prices(height: Height) -> Result> { info!("Fetching BRK height {height} prices..."); retry( @@ -56,7 +56,7 @@ impl BRK { let body: Value = minreq::get(url).send()?.json()?; body.as_array() - .context("Expect to be an array")? + .ok_or(Error::Str("Expect to be an array"))? .iter() .map(Self::value_to_ohlc) .collect::, _>>() @@ -66,7 +66,7 @@ impl BRK { ) } - pub fn get_from_date(&mut self, date: Date) -> color_eyre::Result { + pub fn get_from_date(&mut self, date: Date) -> Result { let dateindex = DateIndex::try_from(date)?; let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap(); @@ -88,10 +88,10 @@ impl BRK { .unwrap() .get(usize::from(dateindex.checked_sub(key).unwrap())) .cloned() - .ok_or(eyre!("Couldn't find date in BRK")) + .ok_or(Error::Str("Couldn't find date in BRK")) } - fn fetch_date_prices(dateindex: DateIndex) -> color_eyre::Result> { + fn fetch_date_prices(dateindex: DateIndex) -> Result> { info!("Fetching BRK dateindex {dateindex} prices..."); retry( @@ -105,7 +105,7 @@ impl BRK { let body: Value = minreq::get(url).send()?.json()?; body.as_array() - .context("Expect to be an array")? + .ok_or(Error::Str("Expect to be an array"))? .iter() .map(Self::value_to_ohlc) .collect::, _>>() @@ -115,15 +115,17 @@ impl BRK { ) } - fn value_to_ohlc(value: &Value) -> color_eyre::Result { - let ohlc = value.as_array().context("Expect as_array to work")?; + fn value_to_ohlc(value: &Value) -> Result { + let ohlc = value + .as_array() + .ok_or(Error::Str("Expect as_array to work"))?; - let get_value = |index: usize| -> color_eyre::Result<_> { + let get_value = |index: usize| -> Result<_> { Ok(Cents::from(Dollars::from( ohlc.get(index) - .context("Expect index key to work")? + .ok_or(Error::Str("Expect index key to work"))? .as_f64() - .context("Expect as_f64 to work")?, + .ok_or(Error::Str("Expect as_f64 to work"))?, ))) }; diff --git a/crates/brk_fetcher/src/kraken.rs b/crates/brk_fetcher/src/kraken.rs index 9193dac4d..99461e970 100644 --- a/crates/brk_fetcher/src/kraken.rs +++ b/crates/brk_fetcher/src/kraken.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; -use brk_core::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp}; -use color_eyre::eyre::ContextCompat; +use brk_error::{Error, Result}; +use brk_structs::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp}; use log::info; use serde_json::Value; @@ -18,7 +18,7 @@ impl Kraken { &mut self, timestamp: Timestamp, previous_timestamp: Option, - ) -> color_eyre::Result { + ) -> Result { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { @@ -32,7 +32,7 @@ impl Kraken { ) } - pub fn fetch_1mn() -> color_eyre::Result> { + pub fn fetch_1mn() -> Result> { info!("Fetching 1mn prices from Kraken..."); retry( @@ -42,7 +42,7 @@ impl Kraken { ) } - pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { + pub fn get_from_1d(&mut self, date: &Date) -> Result { if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date { self._1d.replace(Kraken::fetch_1d()?); } @@ -51,10 +51,10 @@ impl Kraken { .unwrap() .get(date) .cloned() - .ok_or(color_eyre::eyre::Error::msg("Couldn't find date")) + .ok_or(Error::Str("Couldn't find date")) } - pub fn fetch_1d() -> color_eyre::Result> { + pub fn fetch_1d() -> Result> { info!("Fetching daily prices from Kraken..."); retry( @@ -64,38 +64,36 @@ impl Kraken { ) } - fn json_to_timestamp_to_ohlc( - json: &Value, - ) -> color_eyre::Result> { + fn json_to_timestamp_to_ohlc(json: &Value) -> Result> { Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc) } - fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { + fn json_to_date_to_ohlc(json: &Value) -> Result> { Self::json_to_btree(json, Self::array_to_date_and_ohlc) } - fn json_to_btree(json: &Value, fun: F) -> color_eyre::Result> + fn json_to_btree(json: &Value, fun: F) -> Result> where - F: Fn(&Value) -> color_eyre::Result<(K, V)>, + F: Fn(&Value) -> Result<(K, V)>, K: Ord, { json.as_object() - .context("Expect to be an object")? + .ok_or(Error::Str("Expect to be an object"))? .get("result") - .context("Expect object to have result")? + .ok_or(Error::Str("Expect object to have result"))? .as_object() - .context("Expect to be an object")? + .ok_or(Error::Str("Expect to be an object"))? .get("XXBTZUSD") - .context("Expect to have XXBTZUSD")? + .ok_or(Error::Str("Expect to have XXBTZUSD"))? .as_array() - .context("Expect to be an array")? + .ok_or(Error::Str("Expect to be an array"))? .iter() .map(fun) .collect::, _>>() } - fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> { - let array = array.as_array().context("Expect to be array")?; + fn array_to_timestamp_and_ohlc(array: &Value) -> Result<(Timestamp, OHLCCents)> { + let array = array.as_array().ok_or(Error::Str("Expect to be array"))?; let timestamp = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32); @@ -122,7 +120,7 @@ impl Kraken { )) } - fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> { + fn array_to_date_and_ohlc(array: &Value) -> Result<(Date, OHLCCents)> { Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc)) } diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index 32d09b810..c818008b2 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -5,8 +5,8 @@ use std::{collections::BTreeMap, path::Path, thread::sleep, time::Duration}; -use brk_core::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp}; -use color_eyre::eyre::Error; +use brk_error::{Error, Result}; +use brk_structs::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp}; use log::info; mod binance; @@ -29,7 +29,7 @@ pub struct Fetcher { } impl Fetcher { - pub fn import(hars_path: Option<&Path>) -> color_eyre::Result { + pub fn import(hars_path: Option<&Path>) -> Result { Ok(Self { binance: Binance::init(hars_path), kraken: Kraken::default(), @@ -37,11 +37,11 @@ impl Fetcher { }) } - pub fn get_date(&mut self, date: Date) -> color_eyre::Result { + pub fn get_date(&mut self, date: Date) -> Result { self.get_date_(date, 0) } - fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result { + fn get_date_(&mut self, date: Date, tries: usize) -> Result { self.kraken .get_from_1d(&date) .or_else(|_| { @@ -72,7 +72,7 @@ impl Fetcher { height: Height, timestamp: Timestamp, previous_timestamp: Option, - ) -> color_eyre::Result { + ) -> Result { self.get_height_(height, timestamp, previous_timestamp, 0) } @@ -82,7 +82,7 @@ impl Fetcher { timestamp: Timestamp, previous_timestamp: Option, tries: usize, - ) -> color_eyre::Result { + ) -> Result { let timestamp = timestamp.floor_seconds(); if previous_timestamp.is_none() && height != Height::ZERO { @@ -149,7 +149,7 @@ How to fix this: timestamp: Timestamp, previous_timestamp: Option, name: &str, - ) -> color_eyre::Result { + ) -> Result { let previous_ohlc = previous_timestamp .map_or(Some(OHLCCents::default()), |previous_timestamp| { tree.get(&previous_timestamp).cloned() @@ -158,7 +158,7 @@ How to fix this: let last_ohlc = tree.get(×tamp); if previous_ohlc.is_none() || last_ohlc.is_none() { - return Err(Error::msg(format!("Couldn't find timestamp in {name}"))); + return Err(Error::String(format!("Couldn't find timestamp in {name}"))); } let previous_ohlc = previous_ohlc.unwrap(); diff --git a/crates/brk_fetcher/src/retry.rs b/crates/brk_fetcher/src/retry.rs index b1a607031..df8ccaabe 100644 --- a/crates/brk_fetcher/src/retry.rs +++ b/crates/brk_fetcher/src/retry.rs @@ -1,12 +1,9 @@ use std::{fmt::Debug, thread::sleep, time::Duration}; +use brk_error::Result; use log::info; -pub fn retry( - function: impl Fn(usize) -> color_eyre::Result, - sleep_in_s: u64, - retries: usize, -) -> color_eyre::Result +pub fn retry(function: impl Fn(usize) -> Result, sleep_in_s: u64, retries: usize) -> Result where T: Debug, { diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index d3e1aadf3..04cd81972 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -10,13 +10,12 @@ repository.workspace = true [dependencies] bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } -brk_core = { workspace = true } -brk_exit = { workspace = true } +brk_structs = { workspace = true } +brk_error = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_store = { workspace = true } brk_vecs = { workspace = true } -color-eyre = { workspace = true } fjall = { workspace = true } log = { workspace = true } rayon = { workspace = true } diff --git a/crates/brk_indexer/README.md b/crates/brk_indexer/README.md index 4d2bcf41e..889d61bad 100644 --- a/crates/brk_indexer/README.md +++ b/crates/brk_indexer/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../brk_parser`) and creates a database of the vecs (`brk_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code. diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index f92927776..6a1da2142 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -5,20 +5,19 @@ use std::{ time::{Duration, Instant}, }; -use brk_exit::Exit; +use brk_error::Result; use brk_indexer::Indexer; use brk_parser::Parser; +use brk_vecs::Exit; -fn main() -> color_eyre::Result<()> { - color_eyre::install()?; - +fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); - let bitcoin_dir = brk_core::default_bitcoin_path(); - // let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); + // let bitcoin_dir = brk_structs::default_bitcoin_path(); + let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); let outputs_dir = Path::new("./_outputs"); fs::create_dir_all(outputs_dir)?; - // let outputs_dir = brk_core::default_brk_path().join("outputs"); + // let outputs_dir = brk_structs::default_brk_path().join("outputs"); // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( @@ -26,6 +25,7 @@ fn main() -> color_eyre::Result<()> { bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), )?)); let exit = Exit::new(); + exit.set_ctrlc_handler(); let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc); diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index b28620b04..917acda47 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -1,13 +1,15 @@ use bitcoincore_rpc::Client; -use brk_core::{ +use brk_error::{Error, Result}; +use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; +use brk_structs::{ BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex, OutputType, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, - Result, TxIndex, TypeIndex, UnknownOutputIndex, + TxIndex, TypeIndex, UnknownOutputIndex, +}; +use brk_vecs::{ + AnyIterableVec, AnyStoredIterableVec, GenericStoredVec, StoredIndex, StoredRaw, VecIterator, }; -use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; -use brk_vecs::{AnyIterableVec, AnyStampedVec, AnyVec, StampedVec, StoredIndex, StoredType}; -use color_eyre::eyre::ContextCompat; use crate::{Stores, Vecs}; @@ -46,6 +48,7 @@ impl Indexes { OutputType::P2WPKH => *self.p2wpkhaddressindex, OutputType::P2WSH => *self.p2wshaddressindex, OutputType::Unknown => *self.unknownoutputindex, + _ => unreachable!(), } } @@ -87,8 +90,8 @@ impl Indexes { } impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { - type Error = color_eyre::Report; - fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result { + type Error = Error; + fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> Result { // Height at which we want to start: min last saved + 1 or 0 let vecs_starting_height = vecs.starting_height(); let stores_starting_height = stores.starting_height(); @@ -123,100 +126,100 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { &vecs.emptyoutputindex_to_txindex, height, ) - .context("")?, + .ok_or(Error::Str(""))?, height, p2msoutputindex: starting_index( &vecs.height_to_first_p2msoutputindex, &vecs.p2msoutputindex_to_txindex, height, ) - .context("")?, + .ok_or(Error::Str(""))?, opreturnindex: starting_index( &vecs.height_to_first_opreturnindex, &vecs.opreturnindex_to_txindex, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2pk33addressindex: starting_index( &vecs.height_to_first_p2pk33addressindex, &vecs.p2pk33addressindex_to_p2pk33bytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2pk65addressindex: starting_index( &vecs.height_to_first_p2pk65addressindex, &vecs.p2pk65addressindex_to_p2pk65bytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2pkhaddressindex: starting_index( &vecs.height_to_first_p2pkhaddressindex, &vecs.p2pkhaddressindex_to_p2pkhbytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2shaddressindex: starting_index( &vecs.height_to_first_p2shaddressindex, &vecs.p2shaddressindex_to_p2shbytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2traddressindex: starting_index( &vecs.height_to_first_p2traddressindex, &vecs.p2traddressindex_to_p2trbytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2wpkhaddressindex: starting_index( &vecs.height_to_first_p2wpkhaddressindex, &vecs.p2wpkhaddressindex_to_p2wpkhbytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2wshaddressindex: starting_index( &vecs.height_to_first_p2wshaddressindex, &vecs.p2wshaddressindex_to_p2wshbytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, p2aaddressindex: starting_index( &vecs.height_to_first_p2aaddressindex, &vecs.p2aaddressindex_to_p2abytes, height, ) - .context("")?, + .ok_or(Error::Str(""))?, txindex: starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height) - .context("")?, + .ok_or(Error::Str(""))?, inputindex: starting_index( &vecs.height_to_first_inputindex, &vecs.inputindex_to_outputindex, height, ) - .context("")?, + .ok_or(Error::Str(""))?, outputindex: starting_index( &vecs.height_to_first_outputindex, &vecs.outputindex_to_value, height, ) - .context("")?, + .ok_or(Error::Str(""))?, unknownoutputindex: starting_index( &vecs.height_to_first_unknownoutputindex, &vecs.unknownoutputindex_to_txindex, height, ) - .context("")?, + .ok_or(Error::Str(""))?, }) } } pub fn starting_index( - height_to_index: &StampedVec, - index_to_else: &StampedVec, + height_to_index: &impl AnyStoredIterableVec, + index_to_else: &impl AnyIterableVec, starting_height: Height, ) -> Option where - I: StoredType + StoredIndex + From, - T: StoredType, + I: StoredRaw + StoredIndex + From, + T: StoredRaw, { let h = Height::from(u64::from(height_to_index.stamp())); if h.is_zero() { diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 4aa7edcb9..382f99205 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -6,16 +6,16 @@ use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant}; use bitcoin::{Transaction, TxIn, TxOut}; -use brk_core::{ - AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex, - OutputType, Result, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, - TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit, -}; -use brk_exit::Exit; +use brk_error::{Error, Result}; + use brk_parser::Parser; use brk_store::AnyStore; -use brk_vecs::{AnyVec, File, PAGE_SIZE, Reader, VecIterator}; -use color_eyre::eyre::{ContextCompat, eyre}; +use brk_structs::{ + AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex, + OutputType, Sats, StoredBool, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, + TypeIndexWithOutputindex, Unit, Version, Vin, Vout, +}; +use brk_vecs::{AnyVec, Exit, File, GenericStoredVec, PAGE_SIZE, Reader, VecIterator}; use log::{error, info}; use rayon::prelude::*; mod indexes; @@ -38,9 +38,7 @@ pub struct Indexer { } impl Indexer { - pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result { - setrlimit()?; - + pub fn forced_import(outputs_dir: &Path) -> Result { let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?); let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?; @@ -63,7 +61,7 @@ impl Indexer { rpc: &'static bitcoincore_rpc::Client, exit: &Exit, check_collisions: bool, - ) -> color_eyre::Result { + ) -> Result { let file = self.file.clone(); let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)) @@ -177,7 +175,7 @@ impl Indexer { ); parser.parse(start, end).iter().try_for_each( - |(height, block, blockhash)| -> color_eyre::Result<()> { + |(height, block, blockhash)| -> Result<()> { info!("Indexing block {height}..."); idxs.height = height; @@ -204,7 +202,7 @@ impl Indexer { .is_some_and(|prev_height| *prev_height != height) { error!("BlockHash: {blockhash}"); - return Err(eyre!("Collision, expect prefix to need be set yet")); + return Err(Error::Str("Collision, expect prefix to need be set yet")); } idxs.push_if_needed(vecs)?; @@ -227,7 +225,7 @@ impl Indexer { outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle, ) = thread::scope(|scope| { let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle = - scope.spawn(|| -> color_eyre::Result<_> { + scope.spawn(|| -> Result<_> { block .txdata .iter() @@ -247,7 +245,7 @@ impl Indexer { Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt))) }) - .collect::>>() + .collect::>>() }); let input_source_vec_handle = scope.spawn(|| { @@ -266,7 +264,7 @@ impl Indexer { inputs .into_par_iter() .enumerate() - .map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> color_eyre::Result<(InputIndex, InputSource)> { + .map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> Result<(InputIndex, InputSource)> { let txindex = idxs.txindex + block_txindex; let inputindex = idxs.inputindex + InputIndex::from(block_inputindex); @@ -294,7 +292,7 @@ impl Indexer { let vout = Vout::from(outpoint.vout); let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_mmap)? - .context("Expect outputindex to not be none") + .ok_or(Error::Str("Expect outputindex to not be none")) .inspect_err(|_| { dbg!(outpoint.txid, prev_txindex, vout); })?.into_owned() @@ -306,7 +304,7 @@ impl Indexer { outputindex, )))) }) - .try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> { + .try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> { let (key, value) = tuple?; map.insert(key, value); Ok(map) @@ -337,14 +335,14 @@ impl Indexer { .enumerate() .map( #[allow(clippy::type_complexity)] - |(block_outputindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<( + |(block_outputindex, (block_txindex, vout, txout, tx))| -> Result<( OutputIndex, ( &TxOut, TxIndex, Vout, OutputType, - brk_core::Result, + Result, Option, &Transaction, ), @@ -409,12 +407,12 @@ impl Indexer { .p2aaddressindex_to_p2abytes .get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_mmap)? .map(|v| AddressBytes::from(v.into_owned())), - OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => { + _ => { unreachable!() } }; let prev_addressbytes = - prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?; + prev_addressbytes_opt.as_ref().ok_or(Error::Str("Expect to have addressbytes"))?; if stores.addressbyteshash_to_typeindex.needs(height) && prev_addressbytes != addressbytes @@ -452,7 +450,7 @@ impl Indexer { )) }, ) - .try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> { + .try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> { let (key, value) = tuple?; map.insert(key, value); Ok(map) @@ -476,20 +474,19 @@ impl Indexer { let txid_prefix_to_txid_and_block_txindex_and_prev_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle - .ok() - .context( - "Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join", + .map_err(|_| + Error::Str("Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join") )??; let input_source_vec = input_source_vec_handle - .ok() - .context("Export input_source_vec_handle to join")??; + .map_err(|_| + Error::Str("Export input_source_vec_handle to join") + )??; let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt = outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle - .ok() - .context( - "Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join", + .map_err(|_| + Error::Str("Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join") )?; let outputs_len = outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt.len(); @@ -510,7 +507,7 @@ impl Indexer { outputindex, (txout, txindex, vout, outputtype, addressbytes_res, typeindex_opt, _tx), )| - -> color_eyre::Result<()> { + -> Result<()> { let sats = Sats::from(txout.value); if vout.is_zero() { @@ -580,6 +577,7 @@ impl Indexer { vecs.unknownoutputindex_to_txindex.push_if_needed(idxs.unknownoutputindex, txindex)?; idxs.unknownoutputindex.copy_then_increment() }, + _ => unreachable!() }; if let Ok(addressbytes) = addressbytes_res { @@ -618,7 +616,7 @@ impl Indexer { .into_iter() .map( #[allow(clippy::type_complexity)] - |(inputindex, input_source)| -> color_eyre::Result<( + |(inputindex, input_source)| -> Result<( InputIndex, Vin, TxIndex, OutputIndex )> { match input_source { @@ -634,7 +632,7 @@ impl Indexer { let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex .get(&TxidPrefix::from(&txid)) - .context("txid should be in same block").inspect_err(|_| { + .ok_or(Error::Str("txid should be in same block")).inspect_err(|_| { dbg!(&txid_prefix_to_txid_and_block_txindex_and_prev_txindex); // panic!(); })? @@ -643,7 +641,7 @@ impl Indexer { let prev_outputindex = new_txindexvout_to_outputindex .remove(&(prev_txindex, vout)) - .context("should have found addressindex from same block") + .ok_or(Error::Str("should have found addressindex from same block")) .inspect_err(|_| { dbg!(&new_txindexvout_to_outputindex, txin, prev_txindex, vout, txid); })?; @@ -653,7 +651,7 @@ impl Indexer { } }, ) - .try_for_each(|res| -> color_eyre::Result<()> { + .try_for_each(|res| -> Result<()> { let (inputindex, vin, txindex, outputindex) = res?; if vin.is_zero() { @@ -675,7 +673,7 @@ impl Indexer { txid_prefix_to_txid_and_block_txindex_and_prev_txindex .into_iter() .try_for_each( - |(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> { + |(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> Result<()> { let txindex = idxs.txindex + index; txindex_to_tx_and_txid.insert(txindex, (tx, txid)); @@ -700,7 +698,7 @@ impl Indexer { // Ok if `get` is not par as should happen only twice let prev_txid = txindex_to_txid_iter .get(prev_txindex) - .context("To have txid for txindex") + .ok_or(Error::Str("To have txid for txindex")) .inspect_err(|_| { dbg!(txindex, len); })?; @@ -712,11 +710,13 @@ impl Indexer { let only_known_dup_txids = [ bitcoin::Txid::from_str( "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599", - )? + ) + .unwrap() .into(), bitcoin::Txid::from_str( "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468", - )? + ) + .unwrap() .into(), ]; @@ -724,7 +724,7 @@ impl Indexer { if !is_dup { dbg!(height, txindex, prev_txid, prev_txindex); - return Err(eyre!("Expect none")); + return Err(Error::Str("Expect none")); } } } @@ -737,13 +737,13 @@ impl Indexer { txindex_to_tx_and_txid .into_iter() - .try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> { + .try_for_each(|(txindex, (tx, txid))| -> Result<()> { vecs.txindex_to_txversion.push_if_needed(txindex, tx.version.into())?; vecs.txindex_to_txid.push_if_needed(txindex, txid)?; vecs.txindex_to_rawlocktime.push_if_needed(txindex, tx.lock_time.into())?; vecs.txindex_to_base_size.push_if_needed(txindex, tx.base_size().into())?; vecs.txindex_to_total_size.push_if_needed(txindex, tx.total_size().into())?; - vecs.txindex_to_is_explicitly_rbf.push_if_needed(txindex, tx.is_explicitly_rbf())?; + vecs.txindex_to_is_explicitly_rbf.push_if_needed(txindex, StoredBool::from(tx.is_explicitly_rbf()))?; Ok(()) })?; diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index ed0410bd5..550c2c13a 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -1,11 +1,12 @@ use std::{borrow::Cow, fs, path::Path, thread}; -use brk_core::{ - AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex, - OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, -}; +use brk_error::Result; use brk_store::{AnyStore, Store}; -use brk_vecs::{AnyIterableVec, VecIterator}; +use brk_structs::{ + AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex, + OutputType, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, +}; +use brk_vecs::VecIterator; use fjall::{PersistMode, TransactionalKeyspace}; use rayon::prelude::*; @@ -27,7 +28,7 @@ pub struct Stores { const VERSION: Version = Version::ZERO; impl Stores { - pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result { + pub fn forced_import(path: &Path, version: Version) -> Result { fs::create_dir_all(path)?; let keyspace = match brk_store::open_keyspace(path) { @@ -217,7 +218,7 @@ impl Stores { &mut self, vecs: &mut Vecs, starting_indexes: &Indexes, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if self.addressbyteshash_to_typeindex.is_empty()? && self.blockhashprefix_to_height.is_empty()? && self.txidprefix_to_txindex.is_empty()? diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index 44bab394c..932fab6ec 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -1,14 +1,17 @@ use std::sync::Arc; -use brk_core::{ +use brk_error::Result; +use brk_structs::{ AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex, OutputType, P2AAddressIndex, P2ABytes, P2MSOutputIndex, P2PK33AddressIndex, P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex, P2PKHBytes, P2SHAddressIndex, P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, - RawLockTime, Result, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, + RawLockTime, Sats, StoredBool, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion, Txid, TypeIndex, UnknownOutputIndex, Version, Weight, }; -use brk_vecs::{AnyCollectableVec, AnyStampedVec, File, Format, StampedVec}; +use brk_vecs::{ + AnyCollectableVec, AnyStoredVec, CompressedVec, File, GenericStoredVec, RawVec, Stamp, +}; use rayon::prelude::*; use crate::Indexes; @@ -17,320 +20,276 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - pub emptyoutputindex_to_txindex: StampedVec, - pub height_to_blockhash: StampedVec, - pub height_to_difficulty: StampedVec, - pub height_to_first_emptyoutputindex: StampedVec, - pub height_to_first_inputindex: StampedVec, - pub height_to_first_opreturnindex: StampedVec, - pub height_to_first_outputindex: StampedVec, - pub height_to_first_p2aaddressindex: StampedVec, - pub height_to_first_p2msoutputindex: StampedVec, - pub height_to_first_p2pk33addressindex: StampedVec, - pub height_to_first_p2pk65addressindex: StampedVec, - pub height_to_first_p2pkhaddressindex: StampedVec, - pub height_to_first_p2shaddressindex: StampedVec, - pub height_to_first_p2traddressindex: StampedVec, - pub height_to_first_p2wpkhaddressindex: StampedVec, - pub height_to_first_p2wshaddressindex: StampedVec, - pub height_to_first_txindex: StampedVec, - pub height_to_first_unknownoutputindex: StampedVec, + pub emptyoutputindex_to_txindex: CompressedVec, + pub height_to_blockhash: RawVec, + pub height_to_difficulty: CompressedVec, + pub height_to_first_emptyoutputindex: CompressedVec, + pub height_to_first_inputindex: CompressedVec, + pub height_to_first_opreturnindex: CompressedVec, + pub height_to_first_outputindex: CompressedVec, + pub height_to_first_p2aaddressindex: CompressedVec, + pub height_to_first_p2msoutputindex: CompressedVec, + pub height_to_first_p2pk33addressindex: CompressedVec, + pub height_to_first_p2pk65addressindex: CompressedVec, + pub height_to_first_p2pkhaddressindex: CompressedVec, + pub height_to_first_p2shaddressindex: CompressedVec, + pub height_to_first_p2traddressindex: CompressedVec, + pub height_to_first_p2wpkhaddressindex: CompressedVec, + pub height_to_first_p2wshaddressindex: CompressedVec, + pub height_to_first_txindex: CompressedVec, + pub height_to_first_unknownoutputindex: CompressedVec, /// Doesn't guarantee continuity due to possible reorgs - pub height_to_timestamp: StampedVec, - pub height_to_total_size: StampedVec, - pub height_to_weight: StampedVec, + pub height_to_timestamp: CompressedVec, + pub height_to_total_size: CompressedVec, + pub height_to_weight: CompressedVec, /// If outputindex == Outputindex::MAX then it's coinbase - pub inputindex_to_outputindex: StampedVec, - pub opreturnindex_to_txindex: StampedVec, - pub outputindex_to_outputtype: StampedVec, - pub outputindex_to_typeindex: StampedVec, - pub outputindex_to_value: StampedVec, - pub p2aaddressindex_to_p2abytes: StampedVec, - pub p2msoutputindex_to_txindex: StampedVec, - pub p2pk33addressindex_to_p2pk33bytes: StampedVec, - pub p2pk65addressindex_to_p2pk65bytes: StampedVec, - pub p2pkhaddressindex_to_p2pkhbytes: StampedVec, - pub p2shaddressindex_to_p2shbytes: StampedVec, - pub p2traddressindex_to_p2trbytes: StampedVec, - pub p2wpkhaddressindex_to_p2wpkhbytes: StampedVec, - pub p2wshaddressindex_to_p2wshbytes: StampedVec, - pub txindex_to_base_size: StampedVec, - pub txindex_to_first_inputindex: StampedVec, - pub txindex_to_first_outputindex: StampedVec, - pub txindex_to_is_explicitly_rbf: StampedVec, - pub txindex_to_rawlocktime: StampedVec, - pub txindex_to_total_size: StampedVec, - pub txindex_to_txid: StampedVec, - pub txindex_to_txversion: StampedVec, - pub unknownoutputindex_to_txindex: StampedVec, + pub inputindex_to_outputindex: RawVec, + pub opreturnindex_to_txindex: CompressedVec, + pub outputindex_to_outputtype: RawVec, + pub outputindex_to_typeindex: RawVec, + pub outputindex_to_value: RawVec, + pub p2aaddressindex_to_p2abytes: RawVec, + pub p2msoutputindex_to_txindex: CompressedVec, + pub p2pk33addressindex_to_p2pk33bytes: RawVec, + pub p2pk65addressindex_to_p2pk65bytes: RawVec, + pub p2pkhaddressindex_to_p2pkhbytes: RawVec, + pub p2shaddressindex_to_p2shbytes: RawVec, + pub p2traddressindex_to_p2trbytes: RawVec, + pub p2wpkhaddressindex_to_p2wpkhbytes: RawVec, + pub p2wshaddressindex_to_p2wshbytes: RawVec, + pub txindex_to_base_size: CompressedVec, + pub txindex_to_first_inputindex: CompressedVec, + pub txindex_to_first_outputindex: CompressedVec, + pub txindex_to_is_explicitly_rbf: CompressedVec, + pub txindex_to_rawlocktime: CompressedVec, + pub txindex_to_total_size: CompressedVec, + pub txindex_to_txid: RawVec, + pub txindex_to_txversion: CompressedVec, + pub unknownoutputindex_to_txindex: CompressedVec, } impl Vecs { - pub fn forced_import(file: &Arc, version: Version) -> color_eyre::Result { + pub fn forced_import(file: &Arc, version: Version) -> Result { Ok(Self { - emptyoutputindex_to_txindex: StampedVec::forced_import( + emptyoutputindex_to_txindex: CompressedVec::forced_import( file, "txindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_blockhash: StampedVec::forced_import( + height_to_blockhash: RawVec::forced_import( file, "blockhash", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_difficulty: StampedVec::forced_import( + height_to_difficulty: CompressedVec::forced_import( file, "difficulty", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_emptyoutputindex: StampedVec::forced_import( + height_to_first_emptyoutputindex: CompressedVec::forced_import( file, "first_emptyoutputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_inputindex: StampedVec::forced_import( + height_to_first_inputindex: CompressedVec::forced_import( file, "first_inputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_opreturnindex: StampedVec::forced_import( + height_to_first_opreturnindex: CompressedVec::forced_import( file, "first_opreturnindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_outputindex: StampedVec::forced_import( + height_to_first_outputindex: CompressedVec::forced_import( file, "first_outputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2aaddressindex: StampedVec::forced_import( + height_to_first_p2aaddressindex: CompressedVec::forced_import( file, "first_p2aaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2msoutputindex: StampedVec::forced_import( + height_to_first_p2msoutputindex: CompressedVec::forced_import( file, "first_p2msoutputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2pk33addressindex: StampedVec::forced_import( + height_to_first_p2pk33addressindex: CompressedVec::forced_import( file, "first_p2pk33addressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2pk65addressindex: StampedVec::forced_import( + height_to_first_p2pk65addressindex: CompressedVec::forced_import( file, "first_p2pk65addressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2pkhaddressindex: StampedVec::forced_import( + height_to_first_p2pkhaddressindex: CompressedVec::forced_import( file, "first_p2pkhaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2shaddressindex: StampedVec::forced_import( + height_to_first_p2shaddressindex: CompressedVec::forced_import( file, "first_p2shaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2traddressindex: StampedVec::forced_import( + height_to_first_p2traddressindex: CompressedVec::forced_import( file, "first_p2traddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2wpkhaddressindex: StampedVec::forced_import( + height_to_first_p2wpkhaddressindex: CompressedVec::forced_import( file, "first_p2wpkhaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_p2wshaddressindex: StampedVec::forced_import( + height_to_first_p2wshaddressindex: CompressedVec::forced_import( file, "first_p2wshaddressindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_txindex: StampedVec::forced_import( + height_to_first_txindex: CompressedVec::forced_import( file, "first_txindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_first_unknownoutputindex: StampedVec::forced_import( + height_to_first_unknownoutputindex: CompressedVec::forced_import( file, "first_unknownoutputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_timestamp: StampedVec::forced_import( + height_to_timestamp: CompressedVec::forced_import( file, "timestamp", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_total_size: StampedVec::forced_import( + height_to_total_size: CompressedVec::forced_import( file, "total_size", version + VERSION + Version::ZERO, - Format::Raw, )?, - height_to_weight: StampedVec::forced_import( + height_to_weight: CompressedVec::forced_import( file, "weight", version + VERSION + Version::ZERO, - Format::Raw, )?, - inputindex_to_outputindex: StampedVec::forced_import( + inputindex_to_outputindex: RawVec::forced_import( file, "outputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - opreturnindex_to_txindex: StampedVec::forced_import( + opreturnindex_to_txindex: CompressedVec::forced_import( file, "txindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - outputindex_to_outputtype: StampedVec::forced_import( + outputindex_to_outputtype: RawVec::forced_import( file, "outputtype", version + VERSION + Version::ZERO, - Format::Raw, )?, - outputindex_to_typeindex: StampedVec::forced_import( + outputindex_to_typeindex: RawVec::forced_import( file, "typeindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - outputindex_to_value: StampedVec::forced_import( + outputindex_to_value: RawVec::forced_import( file, "value", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2aaddressindex_to_p2abytes: StampedVec::forced_import( + p2aaddressindex_to_p2abytes: RawVec::forced_import( file, "p2abytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2msoutputindex_to_txindex: StampedVec::forced_import( + p2msoutputindex_to_txindex: CompressedVec::forced_import( file, "txindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2pk33addressindex_to_p2pk33bytes: StampedVec::forced_import( + p2pk33addressindex_to_p2pk33bytes: RawVec::forced_import( file, "p2pk33bytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2pk65addressindex_to_p2pk65bytes: StampedVec::forced_import( + p2pk65addressindex_to_p2pk65bytes: RawVec::forced_import( file, "p2pk65bytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2pkhaddressindex_to_p2pkhbytes: StampedVec::forced_import( + p2pkhaddressindex_to_p2pkhbytes: RawVec::forced_import( file, "p2pkhbytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2shaddressindex_to_p2shbytes: StampedVec::forced_import( + p2shaddressindex_to_p2shbytes: RawVec::forced_import( file, "p2shbytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2traddressindex_to_p2trbytes: StampedVec::forced_import( + p2traddressindex_to_p2trbytes: RawVec::forced_import( file, "p2trbytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2wpkhaddressindex_to_p2wpkhbytes: StampedVec::forced_import( + p2wpkhaddressindex_to_p2wpkhbytes: RawVec::forced_import( file, "p2wpkhbytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - p2wshaddressindex_to_p2wshbytes: StampedVec::forced_import( + p2wshaddressindex_to_p2wshbytes: RawVec::forced_import( file, "p2wshbytes", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_base_size: StampedVec::forced_import( + txindex_to_base_size: CompressedVec::forced_import( file, "base_size", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_first_inputindex: StampedVec::forced_import( + txindex_to_first_inputindex: CompressedVec::forced_import( file, "first_inputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_first_outputindex: StampedVec::forced_import( + txindex_to_first_outputindex: CompressedVec::forced_import( file, "first_outputindex", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_is_explicitly_rbf: StampedVec::forced_import( + txindex_to_is_explicitly_rbf: CompressedVec::forced_import( file, "is_explicitly_rbf", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_rawlocktime: StampedVec::forced_import( + txindex_to_rawlocktime: CompressedVec::forced_import( file, "rawlocktime", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_total_size: StampedVec::forced_import( + txindex_to_total_size: CompressedVec::forced_import( file, "total_size", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_txid: StampedVec::forced_import( + txindex_to_txid: RawVec::forced_import( file, "txid", version + VERSION + Version::ZERO, - Format::Raw, )?, - txindex_to_txversion: StampedVec::forced_import( + txindex_to_txversion: CompressedVec::forced_import( file, "txversion", version + VERSION + Version::ZERO, - Format::Raw, )?, - unknownoutputindex_to_txindex: StampedVec::forced_import( + unknownoutputindex_to_txindex: CompressedVec::forced_import( file, "txindex", version + VERSION + Version::ZERO, - Format::Raw, )?, }) } @@ -360,89 +319,93 @@ impl Vecs { let stamp = u64::from(saved_height).into(); self.emptyoutputindex_to_txindex - .truncate_if_needed(emptyoutputindex, stamp)?; - self.height_to_blockhash.truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(emptyoutputindex, stamp)?; + self.height_to_blockhash + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_difficulty - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_emptyoutputindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_inputindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_opreturnindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_outputindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2aaddressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2msoutputindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2pk33addressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2pk65addressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2pkhaddressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2shaddressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2traddressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2wpkhaddressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_p2wshaddressindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_txindex - .truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_first_unknownoutputindex - .truncate_if_needed(height, stamp)?; - self.height_to_timestamp.truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; + self.height_to_timestamp + .truncate_if_needed_with_stamp(height, stamp)?; self.height_to_total_size - .truncate_if_needed(height, stamp)?; - self.height_to_weight.truncate_if_needed(height, stamp)?; + .truncate_if_needed_with_stamp(height, stamp)?; + self.height_to_weight + .truncate_if_needed_with_stamp(height, stamp)?; self.inputindex_to_outputindex - .truncate_if_needed(inputindex, stamp)?; + .truncate_if_needed_with_stamp(inputindex, stamp)?; self.opreturnindex_to_txindex - .truncate_if_needed(opreturnindex, stamp)?; + .truncate_if_needed_with_stamp(opreturnindex, stamp)?; self.outputindex_to_outputtype - .truncate_if_needed(outputindex, stamp)?; + .truncate_if_needed_with_stamp(outputindex, stamp)?; self.outputindex_to_typeindex - .truncate_if_needed(outputindex, stamp)?; + .truncate_if_needed_with_stamp(outputindex, stamp)?; self.outputindex_to_value - .truncate_if_needed(outputindex, stamp)?; + .truncate_if_needed_with_stamp(outputindex, stamp)?; self.p2aaddressindex_to_p2abytes - .truncate_if_needed(p2aaddressindex, stamp)?; + .truncate_if_needed_with_stamp(p2aaddressindex, stamp)?; self.p2msoutputindex_to_txindex - .truncate_if_needed(p2msoutputindex, stamp)?; + .truncate_if_needed_with_stamp(p2msoutputindex, stamp)?; self.p2pk33addressindex_to_p2pk33bytes - .truncate_if_needed(p2pk33addressindex, stamp)?; + .truncate_if_needed_with_stamp(p2pk33addressindex, stamp)?; self.p2pk65addressindex_to_p2pk65bytes - .truncate_if_needed(p2pk65addressindex, stamp)?; + .truncate_if_needed_with_stamp(p2pk65addressindex, stamp)?; self.p2pkhaddressindex_to_p2pkhbytes - .truncate_if_needed(p2pkhaddressindex, stamp)?; + .truncate_if_needed_with_stamp(p2pkhaddressindex, stamp)?; self.p2shaddressindex_to_p2shbytes - .truncate_if_needed(p2shaddressindex, stamp)?; + .truncate_if_needed_with_stamp(p2shaddressindex, stamp)?; self.p2traddressindex_to_p2trbytes - .truncate_if_needed(p2traddressindex, stamp)?; + .truncate_if_needed_with_stamp(p2traddressindex, stamp)?; self.p2wpkhaddressindex_to_p2wpkhbytes - .truncate_if_needed(p2wpkhaddressindex, stamp)?; + .truncate_if_needed_with_stamp(p2wpkhaddressindex, stamp)?; self.p2wshaddressindex_to_p2wshbytes - .truncate_if_needed(p2wshaddressindex, stamp)?; + .truncate_if_needed_with_stamp(p2wshaddressindex, stamp)?; self.txindex_to_base_size - .truncate_if_needed(txindex, stamp)?; + .truncate_if_needed_with_stamp(txindex, stamp)?; self.txindex_to_first_inputindex - .truncate_if_needed(txindex, stamp)?; + .truncate_if_needed_with_stamp(txindex, stamp)?; self.txindex_to_first_outputindex - .truncate_if_needed(txindex, stamp)?; + .truncate_if_needed_with_stamp(txindex, stamp)?; self.txindex_to_is_explicitly_rbf - .truncate_if_needed(txindex, stamp)?; + .truncate_if_needed_with_stamp(txindex, stamp)?; self.txindex_to_rawlocktime - .truncate_if_needed(txindex, stamp)?; + .truncate_if_needed_with_stamp(txindex, stamp)?; self.txindex_to_total_size - .truncate_if_needed(txindex, stamp)?; - self.txindex_to_txid.truncate_if_needed(txindex, stamp)?; + .truncate_if_needed_with_stamp(txindex, stamp)?; + self.txindex_to_txid + .truncate_if_needed_with_stamp(txindex, stamp)?; self.txindex_to_txversion - .truncate_if_needed(txindex, stamp)?; + .truncate_if_needed_with_stamp(txindex, stamp)?; self.unknownoutputindex_to_txindex - .truncate_if_needed(unknownoutputindex, stamp)?; + .truncate_if_needed_with_stamp(unknownoutputindex, stamp)?; Ok(()) } @@ -451,35 +414,37 @@ impl Vecs { match bytes { AddressBytes::P2PK65(bytes) => self .p2pk65addressindex_to_p2pk65bytes - .push_if_needed(index.into(), bytes), + .push_if_needed(index.into(), bytes)?, AddressBytes::P2PK33(bytes) => self .p2pk33addressindex_to_p2pk33bytes - .push_if_needed(index.into(), bytes), + .push_if_needed(index.into(), bytes)?, AddressBytes::P2PKH(bytes) => self .p2pkhaddressindex_to_p2pkhbytes - .push_if_needed(index.into(), bytes), + .push_if_needed(index.into(), bytes)?, AddressBytes::P2SH(bytes) => self .p2shaddressindex_to_p2shbytes - .push_if_needed(index.into(), bytes), + .push_if_needed(index.into(), bytes)?, AddressBytes::P2WPKH(bytes) => self .p2wpkhaddressindex_to_p2wpkhbytes - .push_if_needed(index.into(), bytes), + .push_if_needed(index.into(), bytes)?, AddressBytes::P2WSH(bytes) => self .p2wshaddressindex_to_p2wshbytes - .push_if_needed(index.into(), bytes), + .push_if_needed(index.into(), bytes)?, AddressBytes::P2TR(bytes) => self .p2traddressindex_to_p2trbytes - .push_if_needed(index.into(), bytes), + .push_if_needed(index.into(), bytes)?, AddressBytes::P2A(bytes) => self .p2aaddressindex_to_p2abytes - .push_if_needed(index.into(), bytes), - } + .push_if_needed(index.into(), bytes)?, + }; + Ok(()) } pub fn flush(&mut self, height: Height) -> Result<()> { self.mut_vecs() .into_par_iter() - .try_for_each(|vec| vec.flush(u64::from(height).into())) + .try_for_each(|vec| vec.stamped_flush(Stamp::from(u64::from(height))))?; + Ok(()) } pub fn starting_height(&mut self) -> Height { @@ -542,7 +507,7 @@ impl Vecs { ] } - fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStampedVec> { + fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStoredVec> { vec![ &mut self.emptyoutputindex_to_txindex, &mut self.height_to_blockhash, diff --git a/crates/brk_interface/Cargo.toml b/crates/brk_interface/Cargo.toml index c1c52f07b..62b717241 100644 --- a/crates/brk_interface/Cargo.toml +++ b/crates/brk_interface/Cargo.toml @@ -8,11 +8,11 @@ homepage.workspace = true repository.workspace = true [dependencies] -brk_core = { workspace = true } brk_computer = { workspace = true } +brk_error = { workspace = true } brk_indexer = { workspace = true } +brk_structs = { workspace = true } brk_vecs = { workspace = true } -color-eyre = { workspace = true } derive_deref = { workspace = true } schemars = "1.0.4" serde = { workspace = true } diff --git a/crates/brk_interface/README.md b/crates/brk_interface/README.md index 5faeccaf2..dc892ec24 100644 --- a/crates/brk_interface/README.md +++ b/crates/brk_interface/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A crate that searches for datasets from either `brk_indexer` or `brk_computer` according to given parameters. diff --git a/crates/brk_interface/examples/main.rs b/crates/brk_interface/examples/main.rs index 23538a588..f55bd2703 100644 --- a/crates/brk_interface/examples/main.rs +++ b/crates/brk_interface/examples/main.rs @@ -1,20 +1,16 @@ use std::path::Path; use brk_computer::Computer; +use brk_error::Result; use brk_indexer::Indexer; use brk_interface::{Index, Interface, Params, ParamsOpt}; -use brk_vecs::{Computation, Format}; - -pub fn main() -> color_eyre::Result<()> { - color_eyre::install()?; +pub fn main() -> Result<()> { let outputs_dir = Path::new("../../_outputs"); - let format = Format::Compressed; - let indexer = Indexer::forced_import(outputs_dir)?; - let computer = Computer::forced_import(outputs_dir, &indexer, Computation::Lazy, None, format)?; + let computer = Computer::forced_import(outputs_dir, &indexer, None)?; let interface = Interface::build(&indexer, &computer); diff --git a/crates/brk_interface/src/format.rs b/crates/brk_interface/src/format.rs index 0e7445e4c..8d5b7d642 100644 --- a/crates/brk_interface/src/format.rs +++ b/crates/brk_interface/src/format.rs @@ -1,4 +1,4 @@ -use color_eyre::eyre::eyre; +use brk_error::Error; use schemars::JsonSchema; use serde::Deserialize; @@ -16,7 +16,7 @@ pub enum Format { } impl TryFrom> for Format { - type Error = color_eyre::Report; + type Error = Error; fn try_from(value: Option) -> Result { if let Some(value) = value { let value = value.to_lowercase(); @@ -30,10 +30,10 @@ impl TryFrom> for Format { } else if value == "json" { Ok(Self::JSON) } else { - Err(eyre!("Fail")) + Err(Error::Str("Fail")) } } else { - Err(eyre!("Fail")) + Err(Error::Str("Fail")) } } } diff --git a/crates/brk_interface/src/index.rs b/crates/brk_interface/src/index.rs index 4eaf5bb7e..9ab48a65c 100644 --- a/crates/brk_interface/src/index.rs +++ b/crates/brk_interface/src/index.rs @@ -1,15 +1,15 @@ use std::fmt::{self, Debug}; -use brk_core::{ +use brk_error::Error; +use brk_structs::{ DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height, InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, Printable, QuarterIndex, SemesterIndex, TxIndex, UnknownOutputIndex, WeekIndex, YearIndex, }; -use color_eyre::eyre::eyre; use schemars::JsonSchema; -use serde::{Deserialize, de::Error}; +use serde::Deserialize; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, JsonSchema)] pub enum Index { @@ -146,7 +146,7 @@ impl Index { } impl TryFrom<&str> for Index { - type Error = color_eyre::Report; + type Error = Error; fn try_from(value: &str) -> Result { Ok(match value.to_lowercase().as_str() { v if (Self::DateIndex).possible_values().contains(&v) => Self::DateIndex, @@ -186,7 +186,7 @@ impl TryFrom<&str> for Index { v if (Self::UnknownOutputIndex).possible_values().contains(&v) => { Self::UnknownOutputIndex } - _ => return Err(eyre!("Bad index")), + _ => return Err(Error::Str("Bad index")), }) } } @@ -207,7 +207,7 @@ impl<'de> Deserialize<'de> for Index { // dbg!(index); Ok(index) } else { - Err(Error::custom("Bad index")) + Err(serde::de::Error::custom("Bad index")) } } } diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index 7383fd2c9..b48920587 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -6,9 +6,10 @@ use std::collections::BTreeMap; use brk_computer::Computer; -use brk_core::{Height, Result}; +use brk_error::Result; use brk_indexer::Indexer; -use brk_vecs::{AnyCollectableVec, AnyStampedVec}; +use brk_structs::Height; +use brk_vecs::{AnyCollectableVec, AnyStoredVec}; use tabled::settings::Style; mod deser; @@ -88,7 +89,7 @@ impl<'a> Interface<'a> { &self, vecs: Vec<(String, &&dyn AnyCollectableVec)>, params: &ParamsOpt, - ) -> color_eyre::Result { + ) -> Result { let from = params.from().map(|from| { vecs.iter() .map(|(_, v)| v.i64_to_usize(from)) @@ -106,7 +107,7 @@ impl<'a> Interface<'a> { let mut values = vecs .iter() .map(|(_, vec)| -> Result> { - vec.collect_range_serde_json(from, to) + Ok(vec.collect_range_serde_json(from, to)?) }) .collect::>>()?; @@ -179,7 +180,7 @@ impl<'a> Interface<'a> { }) } - pub fn search_and_format(&self, params: Params) -> color_eyre::Result { + pub fn search_and_format(&self, params: Params) -> Result { self.format(self.search(¶ms), ¶ms.rest) } diff --git a/crates/brk_interface/src/vecs.rs b/crates/brk_interface/src/vecs.rs index 7353d1ffa..4054d2d88 100644 --- a/crates/brk_interface/src/vecs.rs +++ b/crates/brk_interface/src/vecs.rs @@ -33,11 +33,7 @@ impl<'a> Vecs<'a> { .into_iter() .for_each(|vec| this.insert(vec)); - computer - .vecs - .vecs() - .into_iter() - .for_each(|vec| this.insert(vec)); + computer.vecs().into_iter().for_each(|vec| this.insert(vec)); let mut ids = this.id_to_index_to_vec.keys().cloned().collect::>(); diff --git a/crates/brk_logger/Cargo.toml b/crates/brk_logger/Cargo.toml index 08a5cf7f2..a67c2941a 100644 --- a/crates/brk_logger/Cargo.toml +++ b/crates/brk_logger/Cargo.toml @@ -8,7 +8,7 @@ homepage.workspace = true repository.workspace = true [dependencies] -color-eyre = { workspace = true } env_logger = "0.11.8" jiff = { workspace = true } log = { workspace = true } +owo-colors = { workspace = true } diff --git a/crates/brk_logger/README.md b/crates/brk_logger/README.md index 39049f802..e141117d6 100644 --- a/crates/brk_logger/README.md +++ b/crates/brk_logger/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A simple crate built on top of [`env_logger`](https://crates.io/crates/env_logger) to display logs from the [`log`](https://crates.io/crates/log) crate in a colorful and clean format. diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index 153c47e87..892f185d0 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -10,9 +10,9 @@ use std::{ path::Path, }; -use color_eyre::owo_colors::OwoColorize; use env_logger::{Builder, Env}; use jiff::{Timestamp, tz}; +use owo_colors::OwoColorize; #[inline] pub fn init(path: Option<&Path>) { diff --git a/crates/brk_parser/Cargo.toml b/crates/brk_parser/Cargo.toml index 3ad7ada81..3fbfa52b8 100644 --- a/crates/brk_parser/Cargo.toml +++ b/crates/brk_parser/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true [dependencies] bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } -brk_core = { workspace = true } +brk_structs = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } derive_deref = { workspace = true } rayon = { workspace = true } diff --git a/crates/brk_parser/README.md b/crates/brk_parser/README.md index 9b06f1329..87c499780 100644 --- a/crates/brk_parser/README.md +++ b/crates/brk_parser/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A very fast and simple Rust library which reads raw block files (*blkXXXXX.dat*) from Bitcoin Core node and creates an iterator over all the requested blocks in sequential order (0, 1, 2, ...). diff --git a/crates/brk_parser/examples/main.rs b/crates/brk_parser/examples/main.rs index 019807203..db87bb84a 100644 --- a/crates/brk_parser/examples/main.rs +++ b/crates/brk_parser/examples/main.rs @@ -1,12 +1,14 @@ +use std::path::Path; + use bitcoincore_rpc::{Auth, Client}; -use brk_core::{Height, default_bitcoin_path, default_brk_path}; use brk_parser::Parser; +use brk_structs::Height; fn main() { let i = std::time::Instant::now(); - let bitcoin_dir = default_bitcoin_path(); - let brk_dir = default_brk_path(); + let bitcoin_dir = Path::new("").join(""); + let brk_dir = Path::new("").join(""); let rpc = Box::leak(Box::new( Client::new( diff --git a/crates/brk_parser/examples/p2a.rs b/crates/brk_parser/examples/p2a.rs index 5fc6d89a7..f1c1340cb 100644 --- a/crates/brk_parser/examples/p2a.rs +++ b/crates/brk_parser/examples/p2a.rs @@ -1,12 +1,14 @@ +use std::path::Path; + use bitcoincore_rpc::{Auth, Client}; -use brk_core::{Height, OutputType, default_bitcoin_path, default_brk_path}; use brk_parser::Parser; +use brk_structs::{Height, OutputType}; fn main() { let i = std::time::Instant::now(); - let bitcoin_dir = default_bitcoin_path(); - let brk_dir = default_brk_path(); + let bitcoin_dir = Path::new("").join(""); + let brk_dir = Path::new("").join(""); let rpc = Box::leak(Box::new( Client::new( diff --git a/crates/brk_parser/src/blk_recap.rs b/crates/brk_parser/src/blk_recap.rs index 0185084ca..cf8b9cf6d 100644 --- a/crates/brk_parser/src/blk_recap.rs +++ b/crates/brk_parser/src/blk_recap.rs @@ -1,6 +1,6 @@ use std::path::Path; -use brk_core::Height; +use brk_structs::Height; use serde::{Deserialize, Serialize}; use crate::path_to_modified_time; diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index aff73767c..b7e8a817e 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -9,7 +9,7 @@ use bitcoin::{Block, BlockHash}; use bitcoincore_rpc::RpcApi; use blk_index_to_blk_path::*; use blk_recap::BlkRecap; -use brk_core::Height; +use brk_structs::Height; use crossbeam::channel::{Receiver, bounded}; use rayon::prelude::*; diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 57e1b6a22..ed2a8239b 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -12,8 +12,7 @@ axum = { workspace = true } bitcoincore-rpc = { workspace = true } brk_bundler = { workspace = true } brk_computer = { workspace = true } -brk_core = { workspace = true } -brk_exit = { workspace = true } +brk_error = { workspace = true } brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_interface = { workspace = true } @@ -23,16 +22,17 @@ brk_parser = { workspace = true } brk_vecs = { workspace = true } clap = { workspace = true } clap_derive = { workspace = true } -color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } +owo-colors = { workspace = true } +quick_cache = "0.6.15" serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } tower-http = { version = "0.6.6", features = ["compression-full", "trace"] } tracing = "0.1.41" -zip = "4.3.0" +zip = { version = "4.3.0", default-features = false, features = ["deflate"] } [package.metadata.cargo-machete] ignored = ["clap"] diff --git a/crates/brk_server/README.md b/crates/brk_server/README.md index 9f92959e7..5a703d781 100644 --- a/crates/brk_server/README.md +++ b/crates/brk_server/README.md @@ -23,12 +23,6 @@ Nostr - - Bluesky - - - X -

A crate that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_computer` and `brk_interface`. diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 266bcd79b..b100fc64e 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -2,42 +2,38 @@ use std::{path::Path, thread::sleep, time::Duration}; use bitcoincore_rpc::RpcApi; use brk_computer::Computer; -use brk_core::{default_bitcoin_path, default_brk_path}; -use brk_exit::Exit; + +use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::Parser; use brk_server::{Server, Website}; -use brk_vecs::{Computation, Format}; - -pub fn main() -> color_eyre::Result<()> { - color_eyre::install()?; +use brk_vecs::Exit; +pub fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); let process = true; - let bitcoin_dir = default_bitcoin_path(); - let brk_dir = default_brk_path(); + let bitcoin_dir = Path::new(""); + let brk_dir = Path::new(""); let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( "http://localhost:8332", bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), )?)); let exit = Exit::new(); + exit.set_ctrlc_handler(); - let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir, rpc); + let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir.to_path_buf(), rpc); let outputs_dir = Path::new("../../_outputs"); - let format = Format::Compressed; - let mut indexer = Indexer::forced_import(outputs_dir)?; let fetcher = Some(Fetcher::import(None)?); - let mut computer = - Computer::forced_import(outputs_dir, &indexer, Computation::Lazy, fetcher, format)?; + let mut computer = Computer::forced_import(outputs_dir, &indexer, fetcher)?; tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -46,7 +42,12 @@ pub fn main() -> color_eyre::Result<()> { let served_indexer = indexer.clone(); let served_computer = computer.clone(); - let server = Server::new(served_indexer, served_computer, Website::Default)?; + let server = Server::new( + served_indexer, + served_computer, + Website::Default, + Path::new(""), + )?; let server = tokio::spawn(async move { server.serve(true, true).await.unwrap(); @@ -58,7 +59,7 @@ pub fn main() -> color_eyre::Result<()> { let starting_indexes = indexer.index(&parser, rpc, &exit, true)?; - computer.compute(&mut indexer, starting_indexes, &exit)?; + computer.compute(&indexer, starting_indexes, &exit)?; while block_count == rpc.get_block_count()? { sleep(Duration::from_secs(1)) @@ -70,5 +71,5 @@ pub fn main() -> color_eyre::Result<()> { server.await.unwrap(); Ok(()) - }) as color_eyre::Result<()> + }) } diff --git a/crates/brk_server/src/api/interface/mod.rs b/crates/brk_server/src/api/interface/mod.rs index f6e18b74d..282af2e52 100644 --- a/crates/brk_server/src/api/interface/mod.rs +++ b/crates/brk_server/src/api/interface/mod.rs @@ -4,8 +4,9 @@ use axum::{ http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, }; +use brk_error::{Error, Result}; use brk_interface::{Format, Output, Params}; -use color_eyre::eyre::eyre; +use brk_vecs::Stamp; use crate::traits::{HeaderMapExtended, ResponseExtended}; @@ -37,7 +38,7 @@ fn req_to_response_res( headers: HeaderMap, Query(params): Query, AppState { interface, .. }: AppState, -) -> color_eyre::Result { +) -> Result { let vecs = interface.search(¶ms); if vecs.is_empty() { @@ -56,11 +57,17 @@ fn req_to_response_res( .sum::(); if weight > MAX_WEIGHT { - return Err(eyre!("Request is too heavy, max weight is {MAX_WEIGHT}")); + return Err(Error::Str( + "Request is too heavy, max weight is {MAX_WEIGHT}", + )); } // TODO: height should be from vec, but good enough for now - let etag = vecs.first().unwrap().1.etag(interface.get_height(), to); + let etag = vecs + .first() + .unwrap() + .1 + .etag(Stamp::from(u64::from(interface.get_height())), to); if headers .get_if_none_match() diff --git a/crates/brk_server/src/files/file.rs b/crates/brk_server/src/files/file.rs index c2da4aeaa..61fab5a00 100644 --- a/crates/brk_server/src/files/file.rs +++ b/crates/brk_server/src/files/file.rs @@ -6,6 +6,7 @@ use axum::{ http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, }; +use brk_error::Result; use log::{error, info}; use crate::{ @@ -73,7 +74,7 @@ fn path_to_response(headers: &HeaderMap, path: &Path) -> Response { } } -fn path_to_response_(headers: &HeaderMap, path: &Path) -> color_eyre::Result { +fn path_to_response_(headers: &HeaderMap, path: &Path) -> Result { let (modified, date) = headers.check_if_modified_since(path)?; if modified == ModifiedState::NotModifiedSince { return Ok(Response::new_not_modified()); diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index 9e8a43671..01ff86d84 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -21,13 +21,13 @@ use axum::{ }; use brk_bundler::bundle; use brk_computer::Computer; -use brk_core::dot_brk_path; +use brk_error::Result; use brk_indexer::Indexer; use brk_interface::Interface; use brk_mcp::route::MCPRoutes; -use color_eyre::owo_colors::OwoColorize; use files::FilesRoutes; use log::{error, info}; +use owo_colors::OwoColorize; use tokio::net::TcpListener; use tower_http::{compression::CompressionLayer, trace::TraceLayer}; @@ -57,13 +57,17 @@ impl AppState { pub const VERSION: &str = env!("CARGO_PKG_VERSION"); const DEV_PATH: &str = "../.."; -const DOWNLOADS: &str = "downloads"; const WEBSITES: &str = "websites"; pub struct Server(AppState); impl Server { - pub fn new(indexer: Indexer, computer: Computer, website: Website) -> color_eyre::Result { + pub fn new( + indexer: Indexer, + computer: Computer, + website: Website, + downloads_path: &Path, + ) -> Result { let indexer = Box::leak(Box::new(indexer)); let computer = Box::leak(Box::new(computer)); let interface = Box::leak(Box::new(Interface::build(indexer, computer))); @@ -74,8 +78,6 @@ impl Server { let websites_path = if fs::exists(&websites_dev_path)? { websites_dev_path } else { - let downloads_path = dot_brk_path().join(DOWNLOADS); - let downloaded_websites_path = downloads_path.join(format!("brk-{VERSION}")).join(WEBSITES); @@ -90,9 +92,9 @@ impl Server { let bytes = response.as_bytes(); let cursor = Cursor::new(bytes); - let mut zip = zip::ZipArchive::new(cursor)?; + let mut zip = zip::ZipArchive::new(cursor).unwrap(); - zip.extract(&downloads_path)?; + zip.extract(downloads_path).unwrap(); } downloaded_websites_path @@ -112,7 +114,7 @@ impl Server { })) } - pub async fn serve(self, watch: bool, mcp: bool) -> color_eyre::Result<()> { + pub async fn serve(self, watch: bool, mcp: bool) -> Result<()> { let state = self.0; if let Some(websites_path) = state.websites_path.clone() { diff --git a/crates/brk_server/src/traits/header_map.rs b/crates/brk_server/src/traits/header_map.rs index 63e7e2c0f..ca10169ca 100644 --- a/crates/brk_server/src/traits/header_map.rs +++ b/crates/brk_server/src/traits/header_map.rs @@ -7,6 +7,7 @@ use axum::http::{ HeaderMap, header::{self, IF_MODIFIED_SINCE, IF_NONE_MATCH}, }; +use brk_error::Result; use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone}; const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT"; @@ -23,12 +24,8 @@ pub trait HeaderMapExtended { fn get_if_none_match(&self) -> Option<&str>; fn get_if_modified_since(&self) -> Option; - fn check_if_modified_since(&self, path: &Path) - -> color_eyre::Result<(ModifiedState, DateTime)>; - fn check_if_modified_since_( - &self, - duration: Duration, - ) -> color_eyre::Result<(ModifiedState, DateTime)>; + fn check_if_modified_since(&self, path: &Path) -> Result<(ModifiedState, DateTime)>; + fn check_if_modified_since_(&self, duration: Duration) -> Result<(ModifiedState, DateTime)>; fn insert_cache_control_must_revalidate(&mut self); fn insert_cache_control_immutable(&mut self); @@ -91,10 +88,7 @@ impl HeaderMapExtended for HeaderMap { self.insert(header::ETAG, etag.parse().unwrap()); } - fn check_if_modified_since( - &self, - path: &Path, - ) -> color_eyre::Result<(ModifiedState, DateTime)> { + fn check_if_modified_since(&self, path: &Path) -> Result<(ModifiedState, DateTime)> { self.check_if_modified_since_( path.metadata()? .modified()? @@ -102,10 +96,7 @@ impl HeaderMapExtended for HeaderMap { ) } - fn check_if_modified_since_( - &self, - duration: Duration, - ) -> color_eyre::Result<(ModifiedState, DateTime)> { + fn check_if_modified_since_(&self, duration: Duration) -> Result<(ModifiedState, DateTime)> { let date = Timestamp::new(duration.as_secs() as i64, 0) .unwrap() .to_zoned(TimeZone::UTC) diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index 0c1e88c30..a61cc431d 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -10,7 +10,8 @@ homepage.workspace = true repository.workspace = true [dependencies] -brk_core = { workspace = true } +brk_error = { workspace = true } +brk_structs = { workspace = true } byteview = { workspace = true } fjall = { workspace = true } log = { workspace = true } diff --git a/crates/brk_store/examples/main.rs b/crates/brk_store/examples/main.rs index 9d216384a..f2dfab14d 100644 --- a/crates/brk_store/examples/main.rs +++ b/crates/brk_store/examples/main.rs @@ -1,11 +1,11 @@ use std::path::Path; -use brk_core::Result; +use brk_error::Result; fn main() -> Result<()> { let p = Path::new("./examples/_fjall"); - let keyspace = brk_store::open_keyspace(p)?; + let _keyspace = brk_store::open_keyspace(p)?; // let mut store: Store = // brk_store::Store::import(&keyspace, p, "n", Version::ZERO, None)?; diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 2f51e5d51..31ba85fe7 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -11,7 +11,8 @@ use std::{ path::Path, }; -use brk_core::{Height, Result, Version}; +use brk_error::Result; +use brk_structs::{Height, Version}; use byteview::ByteView; use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace, diff --git a/crates/brk_store/src/meta.rs b/crates/brk_store/src/meta.rs index 5597afc14..b8f35613b 100644 --- a/crates/brk_store/src/meta.rs +++ b/crates/brk_store/src/meta.rs @@ -3,7 +3,8 @@ use std::{ path::{Path, PathBuf}, }; -use brk_core::{Result, Version}; +use brk_error::Result; +use brk_structs::Version; use fjall::{PersistMode, TransactionalKeyspace, TransactionalPartitionHandle}; use super::Height; diff --git a/crates/brk_store/src/trait.rs b/crates/brk_store/src/trait.rs index 5b6d06499..0bf2f967e 100644 --- a/crates/brk_store/src/trait.rs +++ b/crates/brk_store/src/trait.rs @@ -1,4 +1,5 @@ -use brk_core::{Height, Result, Version}; +use brk_error::Result; +use brk_structs::{Height, Version}; pub trait AnyStore { fn commit(&mut self, height: Height) -> Result<()>; diff --git a/crates/brk_core/Cargo.toml b/crates/brk_structs/Cargo.toml similarity index 80% rename from crates/brk_core/Cargo.toml rename to crates/brk_structs/Cargo.toml index 3532fdd13..09a14e324 100644 --- a/crates/brk_core/Cargo.toml +++ b/crates/brk_structs/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "brk_core" -description = "The Core (Structs and Errors) of the Bitcoin Research Kit" +name = "brk_structs" +description = "Structs used throughout the Bitcoin Research Kit" version.workspace = true edition.workspace = true license.workspace = true @@ -10,13 +10,13 @@ repository.workspace = true [dependencies] bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } +brk_error = {workspace = true} +brk_vecs = {workspace = true} byteview = { workspace = true } derive_deref = { workspace = true } fjall = { workspace = true } jiff = { workspace = true } -log = { workspace = true } rapidhash = "2.0.2" -rlimit = "0.10.2" serde = { workspace = true } serde_bytes = { workspace = true } serde_json = { workspace = true } diff --git a/crates/brk_structs/README.md b/crates/brk_structs/README.md new file mode 100644 index 000000000..4fe182961 --- /dev/null +++ b/crates/brk_structs/README.md @@ -0,0 +1,28 @@ +# BRK Core + +

+ + GitHub Repo stars + + + License + + + Version + + + Documentation + + Size + + Dependency status + + + Discord + + + Nostr + +

+ +A list of structs that are used throughout the project as units, think of `Date`, `Height`, `Sats`, `Txindex` or anything that can be either a key and/or a value of a dataset. diff --git a/crates/brk_core/src/groups/address.rs b/crates/brk_structs/src/groups/address.rs similarity index 100% rename from crates/brk_core/src/groups/address.rs rename to crates/brk_structs/src/groups/address.rs diff --git a/crates/brk_core/src/groups/by_address_type.rs b/crates/brk_structs/src/groups/by_address_type.rs similarity index 100% rename from crates/brk_core/src/groups/by_address_type.rs rename to crates/brk_structs/src/groups/by_address_type.rs diff --git a/crates/brk_core/src/groups/by_age_range.rs b/crates/brk_structs/src/groups/by_age_range.rs similarity index 100% rename from crates/brk_core/src/groups/by_age_range.rs rename to crates/brk_structs/src/groups/by_age_range.rs diff --git a/crates/brk_core/src/groups/by_amount_range.rs b/crates/brk_structs/src/groups/by_amount_range.rs similarity index 100% rename from crates/brk_core/src/groups/by_amount_range.rs rename to crates/brk_structs/src/groups/by_amount_range.rs diff --git a/crates/brk_core/src/groups/by_any_address.rs b/crates/brk_structs/src/groups/by_any_address.rs similarity index 100% rename from crates/brk_core/src/groups/by_any_address.rs rename to crates/brk_structs/src/groups/by_any_address.rs diff --git a/crates/brk_core/src/groups/by_epoch.rs b/crates/brk_structs/src/groups/by_epoch.rs similarity index 100% rename from crates/brk_core/src/groups/by_epoch.rs rename to crates/brk_structs/src/groups/by_epoch.rs diff --git a/crates/brk_core/src/groups/by_ge_amount.rs b/crates/brk_structs/src/groups/by_ge_amount.rs similarity index 100% rename from crates/brk_core/src/groups/by_ge_amount.rs rename to crates/brk_structs/src/groups/by_ge_amount.rs diff --git a/crates/brk_core/src/groups/by_lt_amount.rs b/crates/brk_structs/src/groups/by_lt_amount.rs similarity index 100% rename from crates/brk_core/src/groups/by_lt_amount.rs rename to crates/brk_structs/src/groups/by_lt_amount.rs diff --git a/crates/brk_core/src/groups/by_max_age.rs b/crates/brk_structs/src/groups/by_max_age.rs similarity index 100% rename from crates/brk_core/src/groups/by_max_age.rs rename to crates/brk_structs/src/groups/by_max_age.rs diff --git a/crates/brk_core/src/groups/by_min_age.rs b/crates/brk_structs/src/groups/by_min_age.rs similarity index 100% rename from crates/brk_core/src/groups/by_min_age.rs rename to crates/brk_structs/src/groups/by_min_age.rs diff --git a/crates/brk_core/src/groups/by_spendable_type.rs b/crates/brk_structs/src/groups/by_spendable_type.rs similarity index 100% rename from crates/brk_core/src/groups/by_spendable_type.rs rename to crates/brk_structs/src/groups/by_spendable_type.rs diff --git a/crates/brk_core/src/groups/by_term.rs b/crates/brk_structs/src/groups/by_term.rs similarity index 100% rename from crates/brk_core/src/groups/by_term.rs rename to crates/brk_structs/src/groups/by_term.rs diff --git a/crates/brk_core/src/groups/by_type.rs b/crates/brk_structs/src/groups/by_type.rs similarity index 97% rename from crates/brk_core/src/groups/by_type.rs rename to crates/brk_structs/src/groups/by_type.rs index dd07592d6..12b1ebda9 100644 --- a/crates/brk_core/src/groups/by_type.rs +++ b/crates/brk_structs/src/groups/by_type.rs @@ -25,6 +25,7 @@ impl GroupedByType { OutputType::Empty => &self.spendable.empty, OutputType::Unknown => &self.spendable.unknown, OutputType::OpReturn => &self.unspendable.opreturn, + _ => unreachable!(), } } @@ -42,6 +43,7 @@ impl GroupedByType { OutputType::Unknown => &mut self.spendable.unknown, OutputType::Empty => &mut self.spendable.empty, OutputType::OpReturn => &mut self.unspendable.opreturn, + _ => unreachable!(), } } } diff --git a/crates/brk_core/src/groups/by_unspendable_type.rs b/crates/brk_structs/src/groups/by_unspendable_type.rs similarity index 100% rename from crates/brk_core/src/groups/by_unspendable_type.rs rename to crates/brk_structs/src/groups/by_unspendable_type.rs diff --git a/crates/brk_core/src/groups/by_value.rs b/crates/brk_structs/src/groups/by_value.rs similarity index 100% rename from crates/brk_core/src/groups/by_value.rs rename to crates/brk_structs/src/groups/by_value.rs diff --git a/crates/brk_core/src/groups/filter.rs b/crates/brk_structs/src/groups/filter.rs similarity index 100% rename from crates/brk_core/src/groups/filter.rs rename to crates/brk_structs/src/groups/filter.rs diff --git a/crates/brk_core/src/groups/mod.rs b/crates/brk_structs/src/groups/mod.rs similarity index 100% rename from crates/brk_core/src/groups/mod.rs rename to crates/brk_structs/src/groups/mod.rs diff --git a/crates/brk_core/src/groups/utxo.rs b/crates/brk_structs/src/groups/utxo.rs similarity index 100% rename from crates/brk_core/src/groups/utxo.rs rename to crates/brk_structs/src/groups/utxo.rs diff --git a/crates/brk_core/src/lib.rs b/crates/brk_structs/src/lib.rs similarity index 54% rename from crates/brk_core/src/lib.rs rename to crates/brk_structs/src/lib.rs index ffffbbe39..43e5af20c 100644 --- a/crates/brk_core/src/lib.rs +++ b/crates/brk_structs/src/lib.rs @@ -1,13 +1,9 @@ #![doc = include_str!("../README.md")] -mod error; mod groups; mod structs; -mod traits; -mod utils; -pub use error::*; pub use groups::*; pub use structs::*; -pub use traits::*; -pub use utils::*; + +pub use brk_vecs::{CheckedSub, Exit, Printable, Version}; diff --git a/crates/brk_core/src/structs/addressbytes.rs b/crates/brk_structs/src/structs/addressbytes.rs similarity index 99% rename from crates/brk_core/src/structs/addressbytes.rs rename to crates/brk_structs/src/structs/addressbytes.rs index cc4125608..88c160d1c 100644 --- a/crates/brk_core/src/structs/addressbytes.rs +++ b/crates/brk_structs/src/structs/addressbytes.rs @@ -6,12 +6,11 @@ use bitcoin::{ opcodes, script::Builder, }; +use brk_error::Error; use derive_deref::{Deref, DerefMut}; use serde::{Serialize, Serializer}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::Error; - use super::OutputType; #[derive(Debug, PartialEq, Eq)] @@ -97,6 +96,7 @@ impl TryFrom<(&ScriptBuf, OutputType)> for AddressBytes { OutputType::Unknown => Err(Error::WrongAddressType), OutputType::Empty => Err(Error::WrongAddressType), OutputType::OpReturn => Err(Error::WrongAddressType), + _ => unreachable!(), } } } diff --git a/crates/brk_core/src/structs/addressbyteshash.rs b/crates/brk_structs/src/structs/addressbyteshash.rs similarity index 100% rename from crates/brk_core/src/structs/addressbyteshash.rs rename to crates/brk_structs/src/structs/addressbyteshash.rs diff --git a/crates/brk_core/src/structs/anyaddressindex.rs b/crates/brk_structs/src/structs/anyaddressindex.rs similarity index 100% rename from crates/brk_core/src/structs/anyaddressindex.rs rename to crates/brk_structs/src/structs/anyaddressindex.rs diff --git a/crates/brk_core/src/structs/bitcoin.rs b/crates/brk_structs/src/structs/bitcoin.rs similarity index 93% rename from crates/brk_core/src/structs/bitcoin.rs rename to crates/brk_structs/src/structs/bitcoin.rs index 84257c818..348b7498c 100644 --- a/crates/brk_core/src/structs/bitcoin.rs +++ b/crates/brk_structs/src/structs/bitcoin.rs @@ -3,14 +3,24 @@ use std::{ ops::{Add, AddAssign, Div, Mul}, }; +use brk_vecs::{CheckedSub, StoredCompressed}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::CheckedSub; - use super::{Sats, StoredF64}; -#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +#[derive( + Debug, + Default, + Clone, + Copy, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, + StoredCompressed, +)] pub struct Bitcoin(f64); impl Add for Bitcoin { diff --git a/crates/brk_core/src/structs/blockhash.rs b/crates/brk_structs/src/structs/blockhash.rs similarity index 100% rename from crates/brk_core/src/structs/blockhash.rs rename to crates/brk_structs/src/structs/blockhash.rs diff --git a/crates/brk_core/src/structs/blockhashprefix.rs b/crates/brk_structs/src/structs/blockhashprefix.rs similarity index 100% rename from crates/brk_core/src/structs/blockhashprefix.rs rename to crates/brk_structs/src/structs/blockhashprefix.rs diff --git a/crates/brk_core/src/structs/cents.rs b/crates/brk_structs/src/structs/cents.rs similarity index 97% rename from crates/brk_core/src/structs/cents.rs rename to crates/brk_structs/src/structs/cents.rs index 3e8750dd9..ec391e8b3 100644 --- a/crates/brk_core/src/structs/cents.rs +++ b/crates/brk_structs/src/structs/cents.rs @@ -1,10 +1,9 @@ use std::ops::{Add, Div, Mul}; +use brk_vecs::{CheckedSub, StoredCompressed}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::CheckedSub; - use super::Dollars; #[derive( @@ -21,6 +20,7 @@ use super::Dollars; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct Cents(i64); diff --git a/crates/brk_core/src/structs/date.rs b/crates/brk_structs/src/structs/date.rs similarity index 92% rename from crates/brk_core/src/structs/date.rs rename to crates/brk_structs/src/structs/date.rs index f1fe488be..1c335734c 100644 --- a/crates/brk_core/src/structs/date.rs +++ b/crates/brk_structs/src/structs/date.rs @@ -1,3 +1,4 @@ +use brk_vecs::StoredCompressed; use jiff::{Span, civil::Date as Date_, tz::TimeZone}; use serde::{Serialize, Serializer}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -5,7 +6,18 @@ use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{DateIndex, Timestamp}; #[derive( - Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout, + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + StoredCompressed, )] pub struct Date(u32); diff --git a/crates/brk_core/src/structs/dateindex.rs b/crates/brk_structs/src/structs/dateindex.rs similarity index 94% rename from crates/brk_core/src/structs/dateindex.rs rename to crates/brk_structs/src/structs/dateindex.rs index 1b47506c2..8b1a66d52 100644 --- a/crates/brk_core/src/structs/dateindex.rs +++ b/crates/brk_structs/src/structs/dateindex.rs @@ -3,14 +3,13 @@ use std::{ ops::{Add, Rem}, }; +use brk_error::Error; +use brk_vecs::{CheckedSub, FromCoarserIndex, Printable, StoredCompressed}; use jiff::Span; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{ - CheckedSub, DecadeIndex, Error, FromCoarserIndex, MonthIndex, Printable, QuarterIndex, - SemesterIndex, WeekIndex, YearIndex, -}; +use crate::{DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, WeekIndex, YearIndex}; use super::Date; @@ -28,6 +27,7 @@ use super::Date; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct DateIndex(u16); @@ -41,6 +41,12 @@ impl From for usize { } } +impl From for u64 { + fn from(value: DateIndex) -> Self { + value.0 as u64 + } +} + impl From for DateIndex { fn from(value: usize) -> Self { Self(value as u16) diff --git a/crates/brk_core/src/structs/decadeindex.rs b/crates/brk_structs/src/structs/decadeindex.rs similarity index 84% rename from crates/brk_core/src/structs/decadeindex.rs rename to crates/brk_structs/src/structs/decadeindex.rs index e02a5a7ff..59ec54c00 100644 --- a/crates/brk_core/src/structs/decadeindex.rs +++ b/crates/brk_structs/src/structs/decadeindex.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::{Date, DateIndex, YearIndex}; #[derive( @@ -25,30 +24,25 @@ use super::{Date, DateIndex, YearIndex}; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] -pub struct DecadeIndex(u8); +pub struct DecadeIndex(u16); -impl From for DecadeIndex { - fn from(value: u8) -> Self { +impl From for DecadeIndex { + fn from(value: u16) -> Self { Self(value) } } -impl From for u8 { - fn from(value: DecadeIndex) -> Self { - value.0 - } -} - impl From for u16 { fn from(value: DecadeIndex) -> Self { - value.0 as u16 + value.0 } } impl From for DecadeIndex { fn from(value: usize) -> Self { - Self(value as u8) + Self(value as u16) } } @@ -62,7 +56,7 @@ impl Add for DecadeIndex { type Output = Self; fn add(self, rhs: usize) -> Self::Output { - Self::from(self.0 + rhs as u8) + Self::from(self.0 + rhs as u16) } } @@ -99,7 +93,7 @@ impl From for DecadeIndex { if year < 2000 { panic!("unsupported") } - Self(((year - 2000) / 10) as u8) + Self((year - 2000) / 10) } } @@ -115,7 +109,7 @@ impl From for DecadeIndex { if v == 0 { Self(0) } else { - Self((((v - 1) / 10) + 1) as u8) + Self((((v - 1) / 10) + 1) as u16) } } } diff --git a/crates/brk_core/src/structs/difficultyepoch.rs b/crates/brk_structs/src/structs/difficultyepoch.rs similarity index 95% rename from crates/brk_core/src/structs/difficultyepoch.rs rename to crates/brk_structs/src/structs/difficultyepoch.rs index fa5772a4f..339f6bdb4 100644 --- a/crates/brk_core/src/structs/difficultyepoch.rs +++ b/crates/brk_structs/src/structs/difficultyepoch.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::Height; #[derive( @@ -25,6 +24,7 @@ use super::Height; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] pub struct DifficultyEpoch(u16); diff --git a/crates/brk_core/src/structs/dollars.rs b/crates/brk_structs/src/structs/dollars.rs similarity index 99% rename from crates/brk_core/src/structs/dollars.rs rename to crates/brk_structs/src/structs/dollars.rs index d88b2b1f9..52c8b1f05 100644 --- a/crates/brk_core/src/structs/dollars.rs +++ b/crates/brk_structs/src/structs/dollars.rs @@ -4,12 +4,11 @@ use std::{ ops::{Add, AddAssign, Div, Mul}, }; +use brk_vecs::{CheckedSub, StoredCompressed}; use derive_deref::Deref; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::CheckedSub; - use super::{Bitcoin, Cents, Close, High, Sats, StoredF32, StoredF64}; #[derive( @@ -24,6 +23,7 @@ use super::{Bitcoin, Cents, Close, High, Sats, StoredF32, StoredF64}; KnownLayout, Serialize, Deserialize, + StoredCompressed, )] pub struct Dollars(f64); diff --git a/crates/brk_core/src/structs/emptyaddressdata.rs b/crates/brk_structs/src/structs/emptyaddressdata.rs similarity index 61% rename from crates/brk_core/src/structs/emptyaddressdata.rs rename to crates/brk_structs/src/structs/emptyaddressdata.rs index 59eaa41eb..ebb636f7b 100644 --- a/crates/brk_core/src/structs/emptyaddressdata.rs +++ b/crates/brk_structs/src/structs/emptyaddressdata.rs @@ -1,6 +1,4 @@ -use byteview::ByteView; use serde::Serialize; -use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{LoadedAddressData, Sats}; @@ -27,19 +25,3 @@ impl From<&LoadedAddressData> for EmptyAddressData { } } } - -impl From for EmptyAddressData { - fn from(value: ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() - } -} -impl From for ByteView { - fn from(value: EmptyAddressData) -> Self { - Self::from(&value) - } -} -impl From<&EmptyAddressData> for ByteView { - fn from(value: &EmptyAddressData) -> Self { - Self::new(value.as_bytes()) - } -} diff --git a/crates/brk_core/src/structs/emptyaddressindex.rs b/crates/brk_structs/src/structs/emptyaddressindex.rs similarity index 95% rename from crates/brk_core/src/structs/emptyaddressindex.rs rename to crates/brk_structs/src/structs/emptyaddressindex.rs index 4fa10a01c..b0d38b6b9 100644 --- a/crates/brk_core/src/structs/emptyaddressindex.rs +++ b/crates/brk_structs/src/structs/emptyaddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, diff --git a/crates/brk_core/src/structs/emptyoutputindex.rs b/crates/brk_structs/src/structs/emptyoutputindex.rs similarity index 85% rename from crates/brk_core/src/structs/emptyoutputindex.rs rename to crates/brk_structs/src/structs/emptyoutputindex.rs index c059f293a..2ae1e3c2e 100644 --- a/crates/brk_core/src/structs/emptyoutputindex.rs +++ b/crates/brk_structs/src/structs/emptyoutputindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct EmptyOutputIndex(TypeIndex); impl From for EmptyOutputIndex { @@ -29,6 +31,11 @@ impl From for EmptyOutputIndex { Self(value) } } +impl From for u64 { + fn from(value: EmptyOutputIndex) -> Self { + Self::from(value.0) + } +} impl From for usize { fn from(value: EmptyOutputIndex) -> Self { Self::from(value.0) diff --git a/crates/brk_core/src/structs/feerate.rs b/crates/brk_structs/src/structs/feerate.rs similarity index 87% rename from crates/brk_core/src/structs/feerate.rs rename to crates/brk_structs/src/structs/feerate.rs index d22c10f90..9ffc48c29 100644 --- a/crates/brk_core/src/structs/feerate.rs +++ b/crates/brk_structs/src/structs/feerate.rs @@ -3,16 +3,19 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::StoredCompressed; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::{Sats, StoredUsize}; +use super::{Sats, StoredU64}; -#[derive(Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)] +#[derive( + Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout, StoredCompressed, +)] pub struct Feerate(f32); -impl From<(Sats, StoredUsize)> for Feerate { - fn from((sats, vsize): (Sats, StoredUsize)) -> Self { +impl From<(Sats, StoredU64)> for Feerate { + fn from((sats, vsize): (Sats, StoredU64)) -> Self { Self((f64::from(sats) / f64::from(vsize)) as f32) } } diff --git a/crates/brk_core/src/structs/halvingepoch.rs b/crates/brk_structs/src/structs/halvingepoch.rs similarity index 83% rename from crates/brk_core/src/structs/halvingepoch.rs rename to crates/brk_structs/src/structs/halvingepoch.rs index cb2e3866e..529c18f58 100644 --- a/crates/brk_core/src/structs/halvingepoch.rs +++ b/crates/brk_structs/src/structs/halvingepoch.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::Height; #[derive( @@ -25,24 +24,25 @@ use super::Height; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] -pub struct HalvingEpoch(u8); +pub struct HalvingEpoch(u16); impl HalvingEpoch { - pub fn new(value: u8) -> Self { + pub fn new(value: u16) -> Self { Self(value) } } -impl From for HalvingEpoch { - fn from(value: u8) -> Self { +impl From for HalvingEpoch { + fn from(value: u16) -> Self { Self(value) } } impl From for HalvingEpoch { fn from(value: usize) -> Self { - Self(value as u8) + Self(value as u16) } } @@ -70,13 +70,13 @@ impl Add for HalvingEpoch { type Output = Self; fn add(self, rhs: usize) -> Self::Output { - Self::from(self.0 + rhs as u8) + Self::from(self.0 + rhs as u16) } } impl From for HalvingEpoch { fn from(value: Height) -> Self { - Self((u32::from(value) / 210_000) as u8) + Self((u32::from(value) / 210_000) as u16) } } diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_structs/src/structs/height.rs similarity index 79% rename from crates/brk_core/src/structs/height.rs rename to crates/brk_structs/src/structs/height.rs index 0e9f40581..034e6dc1d 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_structs/src/structs/height.rs @@ -4,20 +4,23 @@ use std::{ }; use bitcoincore_rpc::{Client, RpcApi}; +use brk_vecs::{CheckedSub, Printable, Stamp, StoredCompressed}; use byteview::ByteView; +use derive_deref::Deref; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; +use crate::copy_first_4bytes; -use super::StoredUsize; +use super::StoredU64; #[derive( Debug, Clone, Copy, PartialEq, + Deref, Eq, PartialOrd, Ord, @@ -28,6 +31,7 @@ use super::StoredUsize; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] pub struct Height(u32); @@ -86,6 +90,14 @@ impl Add for Height { } } +impl Add for Height { + type Output = Self; + + fn add(self, rhs: u64) -> Self::Output { + Self::from(self.0 + rhs as u32) + } +} + impl Add for Height { type Output = Self; @@ -118,6 +130,18 @@ impl AddAssign for Height { } } +impl CheckedSub for Height { + fn checked_sub(self, rhs: u64) -> Option { + self.0.checked_sub(rhs as u32).map(Height::from) + } +} + +impl AddAssign for Height { + fn add_assign(&mut self, rhs: u64) { + *self = self.add(rhs); + } +} + impl Rem for Height { type Output = Self; fn rem(self, rhs: Height) -> Self::Output { @@ -150,8 +174,8 @@ impl From for Height { } } -impl From for Height { - fn from(value: StoredUsize) -> Self { +impl From for Height { + fn from(value: StoredU64) -> Self { Self(*value as u32) } } @@ -192,7 +216,7 @@ impl From for Height { } impl TryFrom<&std::path::Path> for Height { - type Error = crate::Error; + type Error = brk_error::Error; fn try_from(value: &std::path::Path) -> Result { Ok(Self::read_from_bytes(std::fs::read(value)?.as_slice())?.to_owned()) } @@ -200,13 +224,27 @@ impl TryFrom<&std::path::Path> for Height { impl From for Height { fn from(value: byteview::ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() + Self(u32::from_be_bytes(copy_first_4bytes(&value).unwrap())) } } -impl From for byteview::ByteView { +impl From for ByteView { fn from(value: Height) -> Self { - Self::new(value.as_bytes()) + Self::new(&value.0.to_be_bytes()) + } +} + +impl From for Height { + fn from(value: Stamp) -> Self { + let u = u64::from(value); + assert!(u <= u32::MAX as u64); + Self(u as u32) + } +} + +impl From for Stamp { + fn from(value: Height) -> Self { + Self::from(value.0 as u64) } } diff --git a/crates/brk_core/src/structs/inputindex.rs b/crates/brk_structs/src/structs/inputindex.rs similarity index 96% rename from crates/brk_core/src/structs/inputindex.rs rename to crates/brk_structs/src/structs/inputindex.rs index 1ed2a0a66..5e2e78e19 100644 --- a/crates/brk_core/src/structs/inputindex.rs +++ b/crates/brk_structs/src/structs/inputindex.rs @@ -1,11 +1,10 @@ use std::ops::{Add, AddAssign}; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::Vin; #[derive( @@ -24,6 +23,7 @@ use super::Vin; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct InputIndex(u64); diff --git a/crates/brk_core/src/structs/loadedaddressdata.rs b/crates/brk_structs/src/structs/loadedaddressdata.rs similarity index 63% rename from crates/brk_core/src/structs/loadedaddressdata.rs rename to crates/brk_structs/src/structs/loadedaddressdata.rs index 667932a56..5ea97bce7 100644 --- a/crates/brk_core/src/structs/loadedaddressdata.rs +++ b/crates/brk_structs/src/structs/loadedaddressdata.rs @@ -1,9 +1,9 @@ -use byteview::ByteView; +use brk_error::{Error, Result}; +use brk_vecs::CheckedSub; use serde::Serialize; -use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats}; +use crate::{Bitcoin, Dollars, EmptyAddressData, Sats}; #[derive(Debug, Default, Clone, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)] #[repr(C)] @@ -74,34 +74,3 @@ impl From<&EmptyAddressData> for LoadedAddressData { } } } - -impl From for LoadedAddressData { - fn from(value: ByteView) -> Self { - Self { - // MUST be same order as impl From<&AddressData> for ByteView - sent: Sats::read_from_bytes(&value[..8]).unwrap(), - received: Sats::read_from_bytes(&value[8..16]).unwrap(), - realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(), - outputs_len: u32::read_from_bytes(&value[24..]).unwrap(), - padding: 0, - } - } -} -impl From for ByteView { - fn from(value: LoadedAddressData) -> Self { - Self::from(&value) - } -} -impl From<&LoadedAddressData> for ByteView { - fn from(value: &LoadedAddressData) -> Self { - Self::new( - &[ - value.sent.as_bytes(), - value.received.as_bytes(), - value.realized_cap.as_bytes(), - value.outputs_len.as_bytes(), - ] - .concat(), - ) - } -} diff --git a/crates/brk_core/src/structs/loadedaddressindex.rs b/crates/brk_structs/src/structs/loadedaddressindex.rs similarity index 95% rename from crates/brk_core/src/structs/loadedaddressindex.rs rename to crates/brk_structs/src/structs/loadedaddressindex.rs index a01c1290e..5b3b398fe 100644 --- a/crates/brk_core/src/structs/loadedaddressindex.rs +++ b/crates/brk_structs/src/structs/loadedaddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, diff --git a/crates/brk_core/src/structs/mod.rs b/crates/brk_structs/src/structs/mod.rs similarity index 73% rename from crates/brk_core/src/structs/mod.rs rename to crates/brk_structs/src/structs/mod.rs index 6a6109ecd..f19493995 100644 --- a/crates/brk_core/src/structs/mod.rs +++ b/crates/brk_structs/src/structs/mod.rs @@ -1,3 +1,5 @@ +use brk_error::{Error, Result}; + mod addressbytes; mod addressbyteshash; mod anyaddressindex; @@ -37,12 +39,13 @@ mod quarterindex; mod rawlocktime; mod sats; mod semesterindex; +mod stored_bool; mod stored_f32; mod stored_f64; +mod stored_u16; mod stored_u32; mod stored_u64; mod stored_u8; -mod stored_usize; mod timestamp; mod txid; mod txidprefix; @@ -52,7 +55,6 @@ mod typeindex; mod typeindex_with_outputindex; mod unit; mod unknownoutputindex; -mod version; mod vin; mod vout; mod weekindex; @@ -98,12 +100,13 @@ pub use quarterindex::*; pub use rawlocktime::*; pub use sats::*; pub use semesterindex::*; +pub use stored_bool::*; pub use stored_f32::*; pub use stored_f64::*; pub use stored_u8::*; +pub use stored_u16::*; pub use stored_u32::*; pub use stored_u64::*; -pub use stored_usize::*; pub use timestamp::*; pub use txid::*; pub use txidprefix::*; @@ -113,11 +116,34 @@ pub use typeindex::*; pub use typeindex_with_outputindex::*; pub use unit::*; pub use unknownoutputindex::*; -pub use version::*; pub use vin::*; pub use vout::*; pub use weekindex::*; pub use weight::*; pub use yearindex::*; -pub use rlimit; +#[allow(clippy::result_unit_err)] +pub fn copy_first_4bytes(slice: &[u8]) -> Result<[u8; 4]> { + let mut buf: [u8; 4] = [0; 4]; + let buf_len = buf.len(); + if slice.len() < buf_len { + return Err(Error::Str("Buffer is too small to convert to 8 bytes")); + } + slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { + buf[i] = *r; + }); + Ok(buf) +} + +#[allow(clippy::result_unit_err)] +pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> { + let mut buf: [u8; 8] = [0; 8]; + let buf_len = buf.len(); + if slice.len() < buf_len { + return Err(Error::Str("Buffer is too small to convert to 8 bytes")); + } + slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { + buf[i] = *r; + }); + Ok(buf) +} diff --git a/crates/brk_core/src/structs/monthindex.rs b/crates/brk_structs/src/structs/monthindex.rs similarity index 91% rename from crates/brk_core/src/structs/monthindex.rs rename to crates/brk_structs/src/structs/monthindex.rs index 94fdfb4ea..4ba14246b 100644 --- a/crates/brk_core/src/structs/monthindex.rs +++ b/crates/brk_structs/src/structs/monthindex.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::{Date, DateIndex, YearIndex}; #[derive( @@ -25,6 +24,7 @@ use super::{Date, DateIndex, YearIndex}; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] pub struct MonthIndex(u16); @@ -46,6 +46,12 @@ impl From for MonthIndex { } } +impl From for u64 { + fn from(value: MonthIndex) -> Self { + value.0 as u64 + } +} + impl From for usize { fn from(value: MonthIndex) -> Self { value.0 as usize diff --git a/crates/brk_core/src/structs/ohlc.rs b/crates/brk_structs/src/structs/ohlc.rs similarity index 98% rename from crates/brk_core/src/structs/ohlc.rs rename to crates/brk_structs/src/structs/ohlc.rs index 4e13773b4..5dce4b5d0 100644 --- a/crates/brk_core/src/structs/ohlc.rs +++ b/crates/brk_structs/src/structs/ohlc.rs @@ -1,5 +1,6 @@ use std::ops::{Add, AddAssign, Div}; +use brk_vecs::StoredCompressed; use derive_deref::{Deref, DerefMut}; use serde::{Serialize, Serializer, ser::SerializeTuple}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -174,8 +175,9 @@ impl From> for OHLCSats { Deref, DerefMut, Serialize, + StoredCompressed, )] -#[repr(C)] +#[repr(transparent)] pub struct Open(T); impl Open { @@ -271,8 +273,9 @@ where Deref, DerefMut, Serialize, + StoredCompressed, )] -#[repr(C)] +#[repr(transparent)] pub struct High(T); impl High { @@ -368,8 +371,9 @@ where Deref, DerefMut, Serialize, + StoredCompressed, )] -#[repr(C)] +#[repr(transparent)] pub struct Low(T); impl Low { @@ -465,8 +469,9 @@ where Deref, DerefMut, Serialize, + StoredCompressed, )] -#[repr(C)] +#[repr(transparent)] pub struct Close(T); impl Close { diff --git a/crates/brk_core/src/structs/opreturnindex.rs b/crates/brk_structs/src/structs/opreturnindex.rs similarity index 85% rename from crates/brk_core/src/structs/opreturnindex.rs rename to crates/brk_structs/src/structs/opreturnindex.rs index b76c69029..6f8c75b86 100644 --- a/crates/brk_core/src/structs/opreturnindex.rs +++ b/crates/brk_structs/src/structs/opreturnindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,8 +23,10 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct OpReturnIndex(TypeIndex); + impl From for OpReturnIndex { fn from(value: TypeIndex) -> Self { Self(value) @@ -34,6 +37,11 @@ impl From for usize { Self::from(*value) } } +impl From for u64 { + fn from(value: OpReturnIndex) -> Self { + Self::from(*value) + } +} impl From for OpReturnIndex { fn from(value: usize) -> Self { Self(TypeIndex::from(value)) diff --git a/crates/brk_core/src/structs/outputindex.rs b/crates/brk_structs/src/structs/outputindex.rs similarity index 89% rename from crates/brk_core/src/structs/outputindex.rs rename to crates/brk_structs/src/structs/outputindex.rs index 80e1e90a9..be9d686e4 100644 --- a/crates/brk_core/src/structs/outputindex.rs +++ b/crates/brk_structs/src/structs/outputindex.rs @@ -1,10 +1,11 @@ use std::ops::{Add, AddAssign}; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; +use crate::copy_first_8bytes; use super::Vout; @@ -24,6 +25,7 @@ use super::Vout; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct OutputIndex(u64); @@ -105,6 +107,12 @@ impl From for usize { } } +impl From<&[u8]> for OutputIndex { + fn from(value: &[u8]) -> Self { + Self(u64::from_be_bytes(copy_first_8bytes(value).unwrap())) + } +} + impl Printable for OutputIndex { fn to_string() -> &'static str { "outputindex" diff --git a/crates/brk_structs/src/structs/outputtype.rs b/crates/brk_structs/src/structs/outputtype.rs new file mode 100644 index 000000000..3a1814b9d --- /dev/null +++ b/crates/brk_structs/src/structs/outputtype.rs @@ -0,0 +1,381 @@ +use bitcoin::{ScriptBuf, opcodes::all::OP_PUSHBYTES_2}; +use serde::Serialize; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, +)] +#[repr(u8)] +pub enum OutputType { + P2PK65, + P2PK33, + P2PKH, + P2MS, + P2SH, + OpReturn, + P2WPKH, + P2WSH, + P2TR, + P2A, + Dummy10, + Dummy11, + Dummy12, + Dummy13, + Dummy14, + Dummy15, + Dummy16, + Dummy17, + Dummy18, + Dummy19, + Dummy20, + Dummy21, + Dummy22, + Dummy23, + Dummy24, + Dummy25, + Dummy26, + Dummy27, + Dummy28, + Dummy29, + Dummy30, + Dummy31, + Dummy32, + Dummy33, + Dummy34, + Dummy35, + Dummy36, + Dummy37, + Dummy38, + Dummy39, + Dummy40, + Dummy41, + Dummy42, + Dummy43, + Dummy44, + Dummy45, + Dummy46, + Dummy47, + Dummy48, + Dummy49, + Dummy50, + Dummy51, + Dummy52, + Dummy53, + Dummy54, + Dummy55, + Dummy56, + Dummy57, + Dummy58, + Dummy59, + Dummy60, + Dummy61, + Dummy62, + Dummy63, + Dummy64, + Dummy65, + Dummy66, + Dummy67, + Dummy68, + Dummy69, + Dummy70, + Dummy71, + Dummy72, + Dummy73, + Dummy74, + Dummy75, + Dummy76, + Dummy77, + Dummy78, + Dummy79, + Dummy80, + Dummy81, + Dummy82, + Dummy83, + Dummy84, + Dummy85, + Dummy86, + Dummy87, + Dummy88, + Dummy89, + Dummy90, + Dummy91, + Dummy92, + Dummy93, + Dummy94, + Dummy95, + Dummy96, + Dummy97, + Dummy98, + Dummy99, + Dummy100, + Dummy101, + Dummy102, + Dummy103, + Dummy104, + Dummy105, + Dummy106, + Dummy107, + Dummy108, + Dummy109, + Dummy110, + Dummy111, + Dummy112, + Dummy113, + Dummy114, + Dummy115, + Dummy116, + Dummy117, + Dummy118, + Dummy119, + Dummy120, + Dummy121, + Dummy122, + Dummy123, + Dummy124, + Dummy125, + Dummy126, + Dummy127, + Dummy128, + Dummy129, + Dummy130, + Dummy131, + Dummy132, + Dummy133, + Dummy134, + Dummy135, + Dummy136, + Dummy137, + Dummy138, + Dummy139, + Dummy140, + Dummy141, + Dummy142, + Dummy143, + Dummy144, + Dummy145, + Dummy146, + Dummy147, + Dummy148, + Dummy149, + Dummy150, + Dummy151, + Dummy152, + Dummy153, + Dummy154, + Dummy155, + Dummy156, + Dummy157, + Dummy158, + Dummy159, + Dummy160, + Dummy161, + Dummy162, + Dummy163, + Dummy164, + Dummy165, + Dummy166, + Dummy167, + Dummy168, + Dummy169, + Dummy170, + Dummy171, + Dummy172, + Dummy173, + Dummy174, + Dummy175, + Dummy176, + Dummy177, + Dummy178, + Dummy179, + Dummy180, + Dummy181, + Dummy182, + Dummy183, + Dummy184, + Dummy185, + Dummy186, + Dummy187, + Dummy188, + Dummy189, + Dummy190, + Dummy191, + Dummy192, + Dummy193, + Dummy194, + Dummy195, + Dummy196, + Dummy197, + Dummy198, + Dummy199, + Dummy200, + Dummy201, + Dummy202, + Dummy203, + Dummy204, + Dummy205, + Dummy206, + Dummy207, + Dummy208, + Dummy209, + Dummy210, + Dummy211, + Dummy212, + Dummy213, + Dummy214, + Dummy215, + Dummy216, + Dummy217, + Dummy218, + Dummy219, + Dummy220, + Dummy221, + Dummy222, + Dummy223, + Dummy224, + Dummy225, + Dummy226, + Dummy227, + Dummy228, + Dummy229, + Dummy230, + Dummy231, + Dummy232, + Dummy233, + Dummy234, + Dummy235, + Dummy236, + Dummy237, + Dummy238, + Dummy239, + Dummy240, + Dummy241, + Dummy242, + Dummy243, + Dummy244, + Dummy245, + Dummy246, + Dummy247, + Dummy248, + Dummy249, + Dummy250, + Dummy251, + Dummy252, + Dummy253, + Empty = 254, + Unknown = 255, +} + +impl OutputType { + pub fn is_spendable(&self) -> bool { + match self { + Self::P2PK65 => true, + Self::P2PK33 => true, + Self::P2PKH => true, + Self::P2MS => true, + Self::P2SH => true, + Self::OpReturn => false, + Self::P2WPKH => true, + Self::P2WSH => true, + Self::P2TR => true, + Self::P2A => true, + Self::Empty => true, + Self::Unknown => true, + _ => unreachable!(), + } + } + + pub fn is_address(&self) -> bool { + match self { + Self::P2PK65 => true, + Self::P2PK33 => true, + Self::P2PKH => true, + Self::P2MS => false, + Self::P2SH => true, + Self::OpReturn => false, + Self::P2WPKH => true, + Self::P2WSH => true, + Self::P2TR => true, + Self::P2A => true, + Self::Empty => false, + Self::Unknown => false, + _ => unreachable!(), + } + } + + pub fn is_not_address(&self) -> bool { + !self.is_address() + } + + pub fn is_unspendable(&self) -> bool { + !self.is_spendable() + } + + pub fn as_vec() -> Vec { + vec![ + Self::P2PK65, + Self::P2PK33, + Self::P2PKH, + Self::P2MS, + Self::P2SH, + Self::OpReturn, + Self::P2WPKH, + Self::P2WSH, + Self::P2TR, + Self::P2A, + Self::Empty, + Self::Unknown, + ] + } +} + +impl From<&ScriptBuf> for OutputType { + fn from(script: &ScriptBuf) -> Self { + if script.is_p2pk() { + let bytes = script.as_bytes(); + + match bytes.len() { + 67 => Self::P2PK65, + 35 => Self::P2PK33, + _ => { + dbg!(bytes); + unreachable!() + } + } + } else if script.is_p2pkh() { + Self::P2PKH + } else if script.is_multisig() { + Self::P2MS + } else if script.is_p2sh() { + Self::P2SH + } else if script.is_op_return() { + Self::OpReturn + } else if script.is_p2wpkh() { + Self::P2WPKH + } else if script.is_p2wsh() { + Self::P2WSH + } else if script.is_p2tr() { + Self::P2TR + } else if script.witness_version() == Some(bitcoin::WitnessVersion::V1) + && script.len() == 4 + && script.as_bytes()[1] == OP_PUSHBYTES_2.to_u8() + && script.as_bytes()[2..4] == [78, 115] + { + Self::P2A + } else if script.is_empty() { + Self::Empty + } else { + Self::Unknown + } + } +} diff --git a/crates/brk_core/src/structs/p2aaddressindex.rs b/crates/brk_structs/src/structs/p2aaddressindex.rs similarity index 88% rename from crates/brk_core/src/structs/p2aaddressindex.rs rename to crates/brk_structs/src/structs/p2aaddressindex.rs index 53fa4ce6a..2842af509 100644 --- a/crates/brk_core/src/structs/p2aaddressindex.rs +++ b/crates/brk_structs/src/structs/p2aaddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2AAddressIndex(TypeIndex); impl From for P2AAddressIndex { @@ -39,6 +41,11 @@ impl From for u32 { Self::from(*value) } } +impl From for u64 { + fn from(value: P2AAddressIndex) -> Self { + Self::from(*value) + } +} impl From for P2AAddressIndex { fn from(value: u32) -> Self { Self(TypeIndex::from(value)) diff --git a/crates/brk_core/src/structs/p2msoutputindex.rs b/crates/brk_structs/src/structs/p2msoutputindex.rs similarity index 85% rename from crates/brk_core/src/structs/p2msoutputindex.rs rename to crates/brk_structs/src/structs/p2msoutputindex.rs index e315d1b64..ad5643a14 100644 --- a/crates/brk_core/src/structs/p2msoutputindex.rs +++ b/crates/brk_structs/src/structs/p2msoutputindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2MSOutputIndex(TypeIndex); impl From for P2MSOutputIndex { @@ -34,6 +36,11 @@ impl From for usize { Self::from(*value) } } +impl From for u64 { + fn from(value: P2MSOutputIndex) -> Self { + Self::from(*value) + } +} impl From for P2MSOutputIndex { fn from(value: usize) -> Self { Self(TypeIndex::from(value)) diff --git a/crates/brk_core/src/structs/p2pk33addressindex.rs b/crates/brk_structs/src/structs/p2pk33addressindex.rs similarity index 88% rename from crates/brk_core/src/structs/p2pk33addressindex.rs rename to crates/brk_structs/src/structs/p2pk33addressindex.rs index c15ce1a17..7a146b395 100644 --- a/crates/brk_core/src/structs/p2pk33addressindex.rs +++ b/crates/brk_structs/src/structs/p2pk33addressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2PK33AddressIndex(TypeIndex); impl From for P2PK33AddressIndex { @@ -39,6 +41,11 @@ impl From for u32 { Self::from(*value) } } +impl From for u64 { + fn from(value: P2PK33AddressIndex) -> Self { + Self::from(*value) + } +} impl From for P2PK33AddressIndex { fn from(value: u32) -> Self { Self(TypeIndex::from(value)) diff --git a/crates/brk_core/src/structs/p2pk65addressindex.rs b/crates/brk_structs/src/structs/p2pk65addressindex.rs similarity index 88% rename from crates/brk_core/src/structs/p2pk65addressindex.rs rename to crates/brk_structs/src/structs/p2pk65addressindex.rs index a5abb8654..fd1b1c4d5 100644 --- a/crates/brk_core/src/structs/p2pk65addressindex.rs +++ b/crates/brk_structs/src/structs/p2pk65addressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2PK65AddressIndex(TypeIndex); impl From for P2PK65AddressIndex { @@ -34,12 +36,17 @@ impl From for TypeIndex { value.0 } } -impl From for usize { +impl From for u32 { fn from(value: P2PK65AddressIndex) -> Self { Self::from(*value) } } -impl From for u32 { +impl From for u64 { + fn from(value: P2PK65AddressIndex) -> Self { + Self::from(*value) + } +} +impl From for usize { fn from(value: P2PK65AddressIndex) -> Self { Self::from(*value) } diff --git a/crates/brk_core/src/structs/p2pkhaddressindex.rs b/crates/brk_structs/src/structs/p2pkhaddressindex.rs similarity index 88% rename from crates/brk_core/src/structs/p2pkhaddressindex.rs rename to crates/brk_structs/src/structs/p2pkhaddressindex.rs index 32c8c074f..2d9e9d0a1 100644 --- a/crates/brk_core/src/structs/p2pkhaddressindex.rs +++ b/crates/brk_structs/src/structs/p2pkhaddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2PKHAddressIndex(TypeIndex); impl From for P2PKHAddressIndex { @@ -39,6 +41,11 @@ impl From for usize { Self::from(*value) } } +impl From for u64 { + fn from(value: P2PKHAddressIndex) -> Self { + Self::from(*value) + } +} impl From for u32 { fn from(value: P2PKHAddressIndex) -> Self { Self::from(*value) diff --git a/crates/brk_core/src/structs/p2shaddressindex.rs b/crates/brk_structs/src/structs/p2shaddressindex.rs similarity index 82% rename from crates/brk_core/src/structs/p2shaddressindex.rs rename to crates/brk_structs/src/structs/p2shaddressindex.rs index 95aa9c09f..d29776d1d 100644 --- a/crates/brk_core/src/structs/p2shaddressindex.rs +++ b/crates/brk_structs/src/structs/p2shaddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2SHAddressIndex(TypeIndex); impl From for P2SHAddressIndex { @@ -39,11 +41,21 @@ impl From for u32 { Self::from(*value) } } +impl From for u64 { + fn from(value: P2SHAddressIndex) -> Self { + Self::from(*value) + } +} impl From for P2SHAddressIndex { fn from(value: u32) -> Self { Self(TypeIndex::from(value)) } } +impl From for P2SHAddressIndex { + fn from(value: u64) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for usize { fn from(value: P2SHAddressIndex) -> Self { Self::from(*value) diff --git a/crates/brk_core/src/structs/p2traddressindex.rs b/crates/brk_structs/src/structs/p2traddressindex.rs similarity index 88% rename from crates/brk_core/src/structs/p2traddressindex.rs rename to crates/brk_structs/src/structs/p2traddressindex.rs index d9ab27c2f..f96951483 100644 --- a/crates/brk_core/src/structs/p2traddressindex.rs +++ b/crates/brk_structs/src/structs/p2traddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2TRAddressIndex(TypeIndex); impl From for P2TRAddressIndex { @@ -39,6 +41,11 @@ impl From for u32 { Self::from(*value) } } +impl From for u64 { + fn from(value: P2TRAddressIndex) -> Self { + Self::from(*value) + } +} impl From for P2TRAddressIndex { fn from(value: u32) -> Self { Self(TypeIndex::from(value)) diff --git a/crates/brk_core/src/structs/p2wpkhaddressindex.rs b/crates/brk_structs/src/structs/p2wpkhaddressindex.rs similarity index 88% rename from crates/brk_core/src/structs/p2wpkhaddressindex.rs rename to crates/brk_structs/src/structs/p2wpkhaddressindex.rs index 6a21f1b44..c1a8b91e3 100644 --- a/crates/brk_core/src/structs/p2wpkhaddressindex.rs +++ b/crates/brk_structs/src/structs/p2wpkhaddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2WPKHAddressIndex(TypeIndex); impl From for P2WPKHAddressIndex { @@ -34,12 +36,17 @@ impl From for TypeIndex { value.0 } } -impl From for usize { +impl From for u32 { fn from(value: P2WPKHAddressIndex) -> Self { Self::from(*value) } } -impl From for u32 { +impl From for u64 { + fn from(value: P2WPKHAddressIndex) -> Self { + Self::from(*value) + } +} +impl From for usize { fn from(value: P2WPKHAddressIndex) -> Self { Self::from(*value) } diff --git a/crates/brk_core/src/structs/p2wshaddressindex.rs b/crates/brk_structs/src/structs/p2wshaddressindex.rs similarity index 88% rename from crates/brk_core/src/structs/p2wshaddressindex.rs rename to crates/brk_structs/src/structs/p2wshaddressindex.rs index 3256d0806..e1586929d 100644 --- a/crates/brk_core/src/structs/p2wshaddressindex.rs +++ b/crates/brk_structs/src/structs/p2wshaddressindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,6 +23,7 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct P2WSHAddressIndex(TypeIndex); impl From for P2WSHAddressIndex { @@ -39,6 +41,11 @@ impl From for u32 { Self::from(*value) } } +impl From for u64 { + fn from(value: P2WSHAddressIndex) -> Self { + Self::from(*value) + } +} impl From for P2WSHAddressIndex { fn from(value: u32) -> Self { Self(TypeIndex::from(value)) diff --git a/crates/brk_core/src/structs/quarterindex.rs b/crates/brk_structs/src/structs/quarterindex.rs similarity index 96% rename from crates/brk_core/src/structs/quarterindex.rs rename to crates/brk_structs/src/structs/quarterindex.rs index 3ace803ae..f2c665f43 100644 --- a/crates/brk_core/src/structs/quarterindex.rs +++ b/crates/brk_structs/src/structs/quarterindex.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::MonthIndex; #[derive( @@ -25,6 +24,7 @@ use super::MonthIndex; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] pub struct QuarterIndex(u16); diff --git a/crates/brk_core/src/structs/rawlocktime.rs b/crates/brk_structs/src/structs/rawlocktime.rs similarity index 76% rename from crates/brk_core/src/structs/rawlocktime.rs rename to crates/brk_structs/src/structs/rawlocktime.rs index 986592749..2e0b13a37 100644 --- a/crates/brk_core/src/structs/rawlocktime.rs +++ b/crates/brk_structs/src/structs/rawlocktime.rs @@ -1,8 +1,11 @@ use bitcoin::absolute::LockTime; +use brk_vecs::StoredCompressed; use serde::Serialize; -use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)] +#[derive( + Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, FromBytes, Serialize, StoredCompressed, +)] pub struct RawLockTime(u32); impl From for RawLockTime { diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_structs/src/structs/sats.rs similarity index 98% rename from crates/brk_core/src/structs/sats.rs rename to crates/brk_structs/src/structs/sats.rs index 2ff58fdd1..8568ea668 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_structs/src/structs/sats.rs @@ -4,11 +4,12 @@ use std::{ }; use bitcoin::Amount; +use brk_vecs::{CheckedSub, StoredCompressed}; use derive_deref::Deref; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, StoredF64}; +use crate::StoredF64; use super::{Bitcoin, Cents, Dollars, Height}; @@ -28,6 +29,7 @@ use super::{Bitcoin, Cents, Dollars, Height}; KnownLayout, Serialize, Deserialize, + StoredCompressed, )] pub struct Sats(u64); diff --git a/crates/brk_core/src/structs/semesterindex.rs b/crates/brk_structs/src/structs/semesterindex.rs similarity index 96% rename from crates/brk_core/src/structs/semesterindex.rs rename to crates/brk_structs/src/structs/semesterindex.rs index 4d3c0f649..bba339b2c 100644 --- a/crates/brk_core/src/structs/semesterindex.rs +++ b/crates/brk_structs/src/structs/semesterindex.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::MonthIndex; #[derive( @@ -25,6 +24,7 @@ use super::MonthIndex; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] pub struct SemesterIndex(u16); diff --git a/crates/brk_structs/src/structs/stored_bool.rs b/crates/brk_structs/src/structs/stored_bool.rs new file mode 100644 index 000000000..7d0328a33 --- /dev/null +++ b/crates/brk_structs/src/structs/stored_bool.rs @@ -0,0 +1,50 @@ +use brk_vecs::{Printable, StoredCompressed}; +use derive_deref::Deref; +use serde::Serialize; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, + Deref, + Clone, + Default, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, + StoredCompressed, +)] +pub struct StoredBool(u16); + +impl StoredBool { + pub const FALSE: Self = Self(0); + pub const TRUE: Self = Self(1); +} + +impl From for StoredBool { + fn from(value: bool) -> Self { + if value { Self(1) } else { Self(0) } + } +} + +impl From for usize { + fn from(value: StoredBool) -> Self { + value.0 as usize + } +} + +impl Printable for StoredBool { + fn to_string() -> &'static str { + "bool" + } + + fn to_possible_strings() -> &'static [&'static str] { + &["bool"] + } +} diff --git a/crates/brk_core/src/structs/stored_f32.rs b/crates/brk_structs/src/structs/stored_f32.rs similarity index 94% rename from crates/brk_core/src/structs/stored_f32.rs rename to crates/brk_structs/src/structs/stored_f32.rs index a50799a93..aba7820fd 100644 --- a/crates/brk_core/src/structs/stored_f32.rs +++ b/crates/brk_structs/src/structs/stored_f32.rs @@ -4,16 +4,25 @@ use std::{ ops::{Add, AddAssign, Div, Mul, Sub}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::{Dollars, StoredF64}; #[derive( - Debug, Deref, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize, + Debug, + Deref, + Default, + Clone, + Copy, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, + StoredCompressed, )] pub struct StoredF32(f32); diff --git a/crates/brk_core/src/structs/stored_f64.rs b/crates/brk_structs/src/structs/stored_f64.rs similarity index 92% rename from crates/brk_core/src/structs/stored_f64.rs rename to crates/brk_structs/src/structs/stored_f64.rs index f22ff993a..5ab306141 100644 --- a/crates/brk_core/src/structs/stored_f64.rs +++ b/crates/brk_structs/src/structs/stored_f64.rs @@ -4,14 +4,25 @@ use std::{ ops::{Add, AddAssign, Div, Mul}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{Bitcoin, CheckedSub, Dollars, Printable}; +use crate::{Bitcoin, Dollars}; #[derive( - Debug, Deref, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize, + Debug, + Deref, + Default, + Clone, + Copy, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, + StoredCompressed, )] pub struct StoredF64(f64); diff --git a/crates/brk_structs/src/structs/stored_u16.rs b/crates/brk_structs/src/structs/stored_u16.rs new file mode 100644 index 000000000..7bec1148d --- /dev/null +++ b/crates/brk_structs/src/structs/stored_u16.rs @@ -0,0 +1,183 @@ +use std::ops::{Add, AddAssign, Div}; + +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; +use derive_deref::Deref; +use serde::Serialize; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use super::{ + EmptyOutputIndex, OpReturnIndex, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, + P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, + P2WSHAddressIndex, UnknownOutputIndex, +}; + +#[derive( + Debug, + Deref, + Clone, + Default, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, + StoredCompressed, +)] +pub struct StoredU16(u16); + +impl StoredU16 { + pub const ZERO: Self = Self(0); + + pub fn new(counter: u16) -> Self { + Self(counter) + } +} + +impl From for StoredU16 { + fn from(value: u16) -> Self { + Self(value) + } +} + +impl From for StoredU16 { + fn from(value: usize) -> Self { + if value > u16::MAX as usize { + panic!("usize too big (value = {value})") + } + Self(value as u16) + } +} + +impl CheckedSub for StoredU16 { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} + +impl Div for StoredU16 { + type Output = Self; + fn div(self, rhs: usize) -> Self::Output { + Self(self.0 / rhs as u16) + } +} + +impl Add for StoredU16 { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl AddAssign for StoredU16 { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + +impl From for StoredU16 { + fn from(value: f64) -> Self { + if value < 0.0 || value > u16::MAX as f64 { + panic!() + } + Self(value as u16) + } +} + +impl From for f64 { + fn from(value: StoredU16) -> Self { + value.0 as f64 + } +} + +impl From for usize { + fn from(value: StoredU16) -> Self { + value.0 as usize + } +} + +impl From for StoredU16 { + fn from(value: P2PK65AddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2PK33AddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2PKHAddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: OpReturnIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2MSOutputIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2SHAddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2WSHAddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2WPKHAddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2TRAddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: P2AAddressIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: UnknownOutputIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl From for StoredU16 { + fn from(value: EmptyOutputIndex) -> Self { + Self::from(usize::from(value)) + } +} + +impl Printable for StoredU16 { + fn to_string() -> &'static str { + "u16" + } + + fn to_possible_strings() -> &'static [&'static str] { + &["u16"] + } +} diff --git a/crates/brk_core/src/structs/stored_u32.rs b/crates/brk_structs/src/structs/stored_u32.rs similarity index 97% rename from crates/brk_core/src/structs/stored_u32.rs rename to crates/brk_structs/src/structs/stored_u32.rs index b1e4b5d96..bdcd99b7b 100644 --- a/crates/brk_core/src/structs/stored_u32.rs +++ b/crates/brk_structs/src/structs/stored_u32.rs @@ -1,11 +1,10 @@ use std::ops::{Add, AddAssign, Div}; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::{ EmptyOutputIndex, OpReturnIndex, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, @@ -27,6 +26,7 @@ use super::{ IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct StoredU32(u32); diff --git a/crates/brk_core/src/structs/stored_usize.rs b/crates/brk_structs/src/structs/stored_u64.rs similarity index 51% rename from crates/brk_core/src/structs/stored_usize.rs rename to crates/brk_structs/src/structs/stored_u64.rs index 18bd22749..8fdecb419 100644 --- a/crates/brk_core/src/structs/stored_usize.rs +++ b/crates/brk_structs/src/structs/stored_u64.rs @@ -1,11 +1,10 @@ use std::ops::{Add, AddAssign, Div}; -use derive_deref::{Deref, DerefMut}; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; +use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::{ DateIndex, EmptyOutputIndex, Height, InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, @@ -15,10 +14,9 @@ use super::{ #[derive( Debug, - Deref, - DerefMut, - Clone, Default, + Deref, + Clone, Copy, PartialEq, Eq, @@ -29,189 +27,203 @@ use super::{ IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] -pub struct StoredUsize(usize); +pub struct StoredU64(u64); -impl StoredUsize { +impl StoredU64 { pub const ZERO: Self = Self(0); - pub fn new(counter: usize) -> Self { + pub fn new(counter: u64) -> Self { Self(counter) } } -impl From for StoredUsize { - fn from(value: usize) -> Self { +impl From for StoredU64 { + fn from(value: u64) -> Self { Self(value) } } -impl From for usize { - fn from(value: StoredUsize) -> Self { - *value + +impl From for u64 { + fn from(value: StoredU64) -> Self { + value.0 } } -impl CheckedSub for StoredUsize { +impl From for usize { + fn from(value: StoredU64) -> Self { + value.0 as usize + } +} + +impl From for StoredU64 { + fn from(value: usize) -> Self { + Self(value as u64) + } +} + +impl CheckedSub for StoredU64 { fn checked_sub(self, rhs: Self) -> Option { self.0.checked_sub(rhs.0).map(Self) } } -impl Div for StoredUsize { +impl Div for StoredU64 { type Output = Self; fn div(self, rhs: usize) -> Self::Output { - Self(self.0 / rhs) + Self(self.0 / rhs as u64) } } -impl Add for StoredUsize { +impl Add for StoredU64 { type Output = Self; fn add(self, rhs: Self) -> Self::Output { Self(self.0 + rhs.0) } } -impl AddAssign for StoredUsize { +impl AddAssign for StoredU64 { fn add_assign(&mut self, rhs: Self) { *self = *self + rhs } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { panic!() } - Self(value as usize) + Self(value as u64) } } -impl From for f64 { - fn from(value: StoredUsize) -> Self { +impl From for f64 { + fn from(value: StoredU64) -> Self { value.0 as f64 } } -impl From for StoredUsize { - fn from(value: Height) -> Self { - Self::from(usize::from(value)) - } -} - -impl From for StoredUsize { - fn from(value: DateIndex) -> Self { - Self::from(usize::from(value)) - } -} - -impl From for StoredUsize { - fn from(value: MonthIndex) -> Self { - Self::from(usize::from(value)) - } -} - -impl From for StoredUsize { - fn from(value: YearIndex) -> Self { - Self::from(usize::from(value)) - } -} - -impl From for StoredUsize { - fn from(value: OutputIndex) -> Self { - Self::from(usize::from(value)) - } -} - -impl From for StoredUsize { - fn from(value: InputIndex) -> Self { - Self::from(usize::from(value)) - } -} - -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: TxIndex) -> Self { - Self::from(usize::from(value)) + Self(*value as u64) } } -impl From for StoredUsize { +impl From for StoredU64 { + fn from(value: InputIndex) -> Self { + Self(*value) + } +} + +impl From for StoredU64 { + fn from(value: Height) -> Self { + Self(*value as u64) + } +} + +impl From for StoredU64 { + fn from(value: OutputIndex) -> Self { + Self(*value) + } +} + +impl From for StoredU64 { + fn from(value: DateIndex) -> Self { + Self::from(u64::from(value)) + } +} + +impl From for StoredU64 { + fn from(value: MonthIndex) -> Self { + Self::from(u64::from(value)) + } +} + +impl From for StoredU64 { + fn from(value: YearIndex) -> Self { + Self::from(u64::from(value)) + } +} + +impl From for StoredU64 { fn from(value: P2PK65AddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2PK33AddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2PKHAddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: OpReturnIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2MSOutputIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2SHAddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2WSHAddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2WPKHAddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2TRAddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: P2AAddressIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: UnknownOutputIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl From for StoredUsize { +impl From for StoredU64 { fn from(value: EmptyOutputIndex) -> Self { - Self::from(usize::from(value)) + Self::from(u64::from(value)) } } -impl Printable for StoredUsize { +impl Printable for StoredU64 { fn to_string() -> &'static str { - "usize" + "u64" } fn to_possible_strings() -> &'static [&'static str] { - &["usize"] + &["u64"] } } diff --git a/crates/brk_core/src/structs/stored_u8.rs b/crates/brk_structs/src/structs/stored_u8.rs similarity index 98% rename from crates/brk_core/src/structs/stored_u8.rs rename to crates/brk_structs/src/structs/stored_u8.rs index 0d36823aa..554be7d26 100644 --- a/crates/brk_core/src/structs/stored_u8.rs +++ b/crates/brk_structs/src/structs/stored_u8.rs @@ -1,11 +1,10 @@ use std::ops::{Add, AddAssign, Div}; +use brk_vecs::{CheckedSub, Printable}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - pub type StoredPhantom = StoredU8; #[derive( diff --git a/crates/brk_core/src/structs/timestamp.rs b/crates/brk_structs/src/structs/timestamp.rs similarity index 98% rename from crates/brk_core/src/structs/timestamp.rs rename to crates/brk_structs/src/structs/timestamp.rs index 67d74dbd6..44b1806a0 100644 --- a/crates/brk_core/src/structs/timestamp.rs +++ b/crates/brk_structs/src/structs/timestamp.rs @@ -1,12 +1,11 @@ use std::ops::{Add, AddAssign, Div}; +use brk_vecs::{CheckedSub, StoredCompressed}; use derive_deref::Deref; use jiff::{civil::date, tz::TimeZone}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::CheckedSub; - use super::Date; #[derive( @@ -23,6 +22,7 @@ use super::Date; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct Timestamp(u32); diff --git a/crates/brk_core/src/structs/txid.rs b/crates/brk_structs/src/structs/txid.rs similarity index 100% rename from crates/brk_core/src/structs/txid.rs rename to crates/brk_structs/src/structs/txid.rs diff --git a/crates/brk_core/src/structs/txidprefix.rs b/crates/brk_structs/src/structs/txidprefix.rs similarity index 100% rename from crates/brk_core/src/structs/txidprefix.rs rename to crates/brk_structs/src/structs/txidprefix.rs diff --git a/crates/brk_core/src/structs/txindex.rs b/crates/brk_structs/src/structs/txindex.rs similarity index 90% rename from crates/brk_core/src/structs/txindex.rs rename to crates/brk_structs/src/structs/txindex.rs index ca2b8f191..f6a9a84d8 100644 --- a/crates/brk_core/src/structs/txindex.rs +++ b/crates/brk_structs/src/structs/txindex.rs @@ -1,12 +1,12 @@ use std::ops::{Add, AddAssign}; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; -use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; +use crate::copy_first_4bytes; use super::StoredU32; @@ -26,6 +26,7 @@ use super::StoredU32; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct TxIndex(u32); @@ -97,12 +98,12 @@ impl From for usize { impl From for TxIndex { fn from(value: ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() + Self(u32::from_be_bytes(copy_first_4bytes(&value).unwrap())) } } impl From for ByteView { fn from(value: TxIndex) -> Self { - Self::new(value.as_bytes()) + Self::new(&value.to_be_bytes()) } } diff --git a/crates/brk_core/src/structs/txversion.rs b/crates/brk_structs/src/structs/txversion.rs similarity index 82% rename from crates/brk_core/src/structs/txversion.rs rename to crates/brk_structs/src/structs/txversion.rs index 2eb5ab4af..8b078f273 100644 --- a/crates/brk_core/src/structs/txversion.rs +++ b/crates/brk_structs/src/structs/txversion.rs @@ -1,8 +1,9 @@ +use brk_vecs::StoredCompressed; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::StoredU8; +use super::StoredU16; #[derive( Debug, @@ -18,14 +19,15 @@ use super::StoredU8; KnownLayout, FromBytes, Serialize, + StoredCompressed, )] -pub struct TxVersion(u8); +pub struct TxVersion(u16); impl TxVersion { pub const ONE: Self = Self(1); pub const TWO: Self = Self(2); pub const THREE: Self = Self(3); - pub const NON_STANDARD: Self = Self(u8::MAX); + pub const NON_STANDARD: Self = Self(u16::MAX); } impl From for TxVersion { @@ -45,7 +47,7 @@ impl From for bitcoin::transaction::Version { } } -impl From for StoredU8 { +impl From for StoredU16 { fn from(value: TxVersion) -> Self { Self::from(value.0) } diff --git a/crates/brk_core/src/structs/typeindex.rs b/crates/brk_structs/src/structs/typeindex.rs similarity index 84% rename from crates/brk_core/src/structs/typeindex.rs rename to crates/brk_structs/src/structs/typeindex.rs index 8cc9236b8..b1be0e57b 100644 --- a/crates/brk_core/src/structs/typeindex.rs +++ b/crates/brk_structs/src/structs/typeindex.rs @@ -1,11 +1,12 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, StoredCompressed}; use byteview::ByteView; use serde::Serialize; -use zerocopy::{FromBytes, IntoBytes}; +use zerocopy::IntoBytes; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::CheckedSub; +use crate::copy_first_4bytes; #[derive( Debug, @@ -21,6 +22,7 @@ use crate::CheckedSub; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct TypeIndex(u32); @@ -92,14 +94,21 @@ impl Add for TypeIndex { Self(self.0 + rhs.0) } } + +impl From<&[u8]> for TypeIndex { + fn from(value: &[u8]) -> Self { + Self(u32::from_be_bytes(copy_first_4bytes(value).unwrap())) + } +} + impl From for TypeIndex { fn from(value: ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() + Self::from(value.as_bytes()) } } impl From for ByteView { fn from(value: TypeIndex) -> Self { - Self::new(value.as_bytes()) + Self::new(&value.0.to_be_bytes()) } } diff --git a/crates/brk_core/src/structs/typeindex_with_outputindex.rs b/crates/brk_structs/src/structs/typeindex_with_outputindex.rs similarity index 79% rename from crates/brk_core/src/structs/typeindex_with_outputindex.rs rename to crates/brk_structs/src/structs/typeindex_with_outputindex.rs index 8db54e167..6aede64a2 100644 --- a/crates/brk_core/src/structs/typeindex_with_outputindex.rs +++ b/crates/brk_structs/src/structs/typeindex_with_outputindex.rs @@ -1,9 +1,7 @@ use byteview::ByteView; use serde::Serialize; -use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; - -use super::OutputIndex; +use super::{OutputIndex, TypeIndex}; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)] pub struct TypeIndexWithOutputindex { @@ -22,8 +20,8 @@ impl From<(TypeIndex, OutputIndex)> for TypeIndexWithOutputindex { impl From for TypeIndexWithOutputindex { fn from(value: ByteView) -> Self { - let typeindex = TypeIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap())); - let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap())); + let typeindex = TypeIndex::from(&value[0..4]); + let outputindex = OutputIndex::from(&value[4..12]); Self { typeindex, outputindex, diff --git a/crates/brk_core/src/structs/unit.rs b/crates/brk_structs/src/structs/unit.rs similarity index 100% rename from crates/brk_core/src/structs/unit.rs rename to crates/brk_structs/src/structs/unit.rs diff --git a/crates/brk_core/src/structs/unknownoutputindex.rs b/crates/brk_structs/src/structs/unknownoutputindex.rs similarity index 85% rename from crates/brk_core/src/structs/unknownoutputindex.rs rename to crates/brk_structs/src/structs/unknownoutputindex.rs index fa80866ad..6926a54c2 100644 --- a/crates/brk_core/src/structs/unknownoutputindex.rs +++ b/crates/brk_structs/src/structs/unknownoutputindex.rs @@ -1,10 +1,11 @@ use std::ops::Add; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable, TypeIndex}; +use crate::TypeIndex; #[derive( Debug, @@ -22,13 +23,20 @@ use crate::{CheckedSub, Printable, TypeIndex}; IntoBytes, KnownLayout, Serialize, + StoredCompressed, )] pub struct UnknownOutputIndex(TypeIndex); + impl From for UnknownOutputIndex { fn from(value: TypeIndex) -> Self { Self(value) } } +impl From for u64 { + fn from(value: UnknownOutputIndex) -> Self { + Self::from(*value) + } +} impl From for usize { fn from(value: UnknownOutputIndex) -> Self { Self::from(*value) diff --git a/crates/brk_core/src/structs/vin.rs b/crates/brk_structs/src/structs/vin.rs similarity index 100% rename from crates/brk_core/src/structs/vin.rs rename to crates/brk_structs/src/structs/vin.rs diff --git a/crates/brk_core/src/structs/vout.rs b/crates/brk_structs/src/structs/vout.rs similarity index 100% rename from crates/brk_core/src/structs/vout.rs rename to crates/brk_structs/src/structs/vout.rs diff --git a/crates/brk_core/src/structs/weekindex.rs b/crates/brk_structs/src/structs/weekindex.rs similarity index 96% rename from crates/brk_core/src/structs/weekindex.rs rename to crates/brk_structs/src/structs/weekindex.rs index d302cdfe7..e603d781d 100644 --- a/crates/brk_core/src/structs/weekindex.rs +++ b/crates/brk_structs/src/structs/weekindex.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::{Date, DateIndex}; #[derive( @@ -25,6 +24,7 @@ use super::{Date, DateIndex}; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] pub struct WeekIndex(u16); diff --git a/crates/brk_core/src/structs/weight.rs b/crates/brk_structs/src/structs/weight.rs similarity index 96% rename from crates/brk_core/src/structs/weight.rs rename to crates/brk_structs/src/structs/weight.rs index 10b8ed8ad..d8230ba4b 100644 --- a/crates/brk_core/src/structs/weight.rs +++ b/crates/brk_structs/src/structs/weight.rs @@ -1,5 +1,6 @@ use std::ops::{Add, AddAssign, Div}; +use brk_vecs::StoredCompressed; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -18,6 +19,7 @@ use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; KnownLayout, FromBytes, Serialize, + StoredCompressed, )] pub struct Weight(u64); diff --git a/crates/brk_core/src/structs/yearindex.rs b/crates/brk_structs/src/structs/yearindex.rs similarity index 80% rename from crates/brk_core/src/structs/yearindex.rs rename to crates/brk_structs/src/structs/yearindex.rs index 8ba72883f..bea3b6e28 100644 --- a/crates/brk_core/src/structs/yearindex.rs +++ b/crates/brk_structs/src/structs/yearindex.rs @@ -3,11 +3,10 @@ use std::{ ops::{Add, AddAssign, Div}, }; +use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Printable}; - use super::{Date, DateIndex, MonthIndex}; #[derive( @@ -25,18 +24,25 @@ use super::{Date, DateIndex, MonthIndex}; Immutable, IntoBytes, KnownLayout, + StoredCompressed, )] -pub struct YearIndex(u8); +pub struct YearIndex(u16); -impl From for YearIndex { - fn from(value: u8) -> Self { +impl From for YearIndex { + fn from(value: u16) -> Self { Self(value) } } impl From for YearIndex { fn from(value: usize) -> Self { - Self(value as u8) + Self(value as u16) + } +} + +impl From for u64 { + fn from(value: YearIndex) -> Self { + value.0 as u64 } } @@ -50,7 +56,7 @@ impl Add for YearIndex { type Output = Self; fn add(self, rhs: usize) -> Self::Output { - Self::from(self.0 + rhs as u8) + Self::from(self.0 + rhs as u16) } } @@ -83,13 +89,13 @@ impl From for YearIndex { impl From for YearIndex { fn from(value: Date) -> Self { - Self((value.year() - 2009) as u8) + Self(value.year() - 2009) } } impl From for u16 { fn from(value: YearIndex) -> Self { - value.0 as u16 + value.0 } } @@ -101,7 +107,7 @@ impl CheckedSub for YearIndex { impl From for YearIndex { fn from(value: MonthIndex) -> Self { - Self((usize::from(value) / 12) as u8) + Self((usize::from(value) / 12) as u16) } } diff --git a/crates/brk_vecs/Cargo.toml b/crates/brk_vecs/Cargo.toml index 96abcc4af..3f707206d 100644 --- a/crates/brk_vecs/Cargo.toml +++ b/crates/brk_vecs/Cargo.toml @@ -10,10 +10,8 @@ homepage.workspace = true repository.workspace = true [dependencies] -brk_core = { workspace = true } -brk_exit = { workspace = true } -clap = { workspace = true } -clap_derive = { workspace = true } +brk_vecs_macros = { workspace = true } +ctrlc = { version = "3.4.7", features = ["termination"] } libc = "0.2.174" log = { workspace = true } memmap2 = "0.9.7" @@ -23,7 +21,4 @@ serde = { workspace = true } serde_json = { workspace = true } zerocopy = { workspace = true } zerocopy-derive = { workspace = true } -zstd = "0.13.3" - -[package.metadata.cargo-machete] -ignored = ["clap"] +pco = "0.4.6" diff --git a/crates/brk_vecs/README.md b/crates/brk_vecs/README.md new file mode 100644 index 000000000..f6e7e869c --- /dev/null +++ b/crates/brk_vecs/README.md @@ -0,0 +1,7 @@ +# Vecs + +- Columnar +- Embedded +- Index as keys, thus free +- Only values are stored +- Raw or Compressed diff --git a/crates/brk_vecs/examples/file.rs b/crates/brk_vecs/examples/file.rs index 05f073ba2..2678c8168 100644 --- a/crates/brk_vecs/examples/file.rs +++ b/crates/brk_vecs/examples/file.rs @@ -1,7 +1,6 @@ use std::{fs, path::Path}; -use brk_core::Result; -use brk_vecs::{File, PAGE_SIZE}; +use brk_vecs::{File, PAGE_SIZE, Result}; fn main() -> Result<()> { let _ = fs::remove_dir_all("vecs"); diff --git a/crates/brk_vecs/examples/raw.rs b/crates/brk_vecs/examples/raw.rs index 8445b8288..fbdd6beae 100644 --- a/crates/brk_vecs/examples/raw.rs +++ b/crates/brk_vecs/examples/raw.rs @@ -1,11 +1,12 @@ use std::{fs, path::Path, sync::Arc}; -use brk_core::{DateIndex, Version}; -use brk_vecs::{AnyVec, CollectableVec, File, GenericStoredVec, RawVec, Stamp, VecIterator}; +use brk_vecs::{ + AnyStoredVec, AnyVec, CollectableVec, File, GenericStoredVec, RawVec, Stamp, VecIterator, + Version, +}; -type I = DateIndex; #[allow(clippy::upper_case_acronyms)] -type VEC = RawVec; +type VEC = RawVec; fn main() -> Result<(), Box> { let _ = fs::remove_dir_all("raw"); @@ -23,11 +24,11 @@ fn main() -> Result<(), Box> { }); let mut iter = vec.into_iter(); - dbg!(iter.get(0.into())); - dbg!(iter.get(1.into())); - dbg!(iter.get(2.into())); - dbg!(iter.get(20.into())); - dbg!(iter.get(21.into())); + dbg!(iter.get(0)); + dbg!(iter.get(1)); + dbg!(iter.get(2)); + dbg!(iter.get(20)); + dbg!(iter.get(21)); drop(iter); vec.flush()?; @@ -41,25 +42,25 @@ fn main() -> Result<(), Box> { vec.mut_header().update_stamp(Stamp::new(100)); let mut iter = vec.into_iter(); - dbg!(iter.get(0.into())); - dbg!(iter.get(1.into())); - dbg!(iter.get(2.into())); - dbg!(iter.get(3.into())); - dbg!(iter.get(4.into())); - dbg!(iter.get(5.into())); - dbg!(iter.get(20.into())); - dbg!(iter.get(20.into())); - dbg!(iter.get(0.into())); + dbg!(iter.get(0)); + dbg!(iter.get(1)); + dbg!(iter.get(2)); + dbg!(iter.get(3)); + dbg!(iter.get(4)); + dbg!(iter.get(5)); + dbg!(iter.get(20)); + dbg!(iter.get(20)); + dbg!(iter.get(0)); drop(iter); vec.push(21); vec.push(22); let mut iter = vec.into_iter(); - dbg!(iter.get(20.into())); - dbg!(iter.get(21.into())); - dbg!(iter.get(22.into())); - dbg!(iter.get(23.into())); + dbg!(iter.get(20)); + dbg!(iter.get(21)); + dbg!(iter.get(22)); + dbg!(iter.get(23)); drop(iter); vec.flush()?; @@ -69,18 +70,18 @@ fn main() -> Result<(), Box> { let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; let mut iter = vec.into_iter(); - dbg!(iter.get(0.into())); - dbg!(iter.get(20.into())); - dbg!(iter.get(21.into())); - dbg!(iter.get(22.into())); + dbg!(iter.get(0)); + dbg!(iter.get(20)); + dbg!(iter.get(21)); + dbg!(iter.get(22)); drop(iter); - vec.truncate_if_needed(14.into())?; + vec.truncate_if_needed(14)?; let mut iter = vec.into_iter(); - dbg!(iter.get(0.into())); - dbg!(iter.get(5.into())); - dbg!(iter.get(20.into())); + dbg!(iter.get(0)); + dbg!(iter.get(5)); + dbg!(iter.get(20)); drop(iter); dbg!(vec.collect_signed_range(Some(-5), None)?); @@ -103,14 +104,14 @@ fn main() -> Result<(), Box> { }); let mut iter = vec.into_iter(); - dbg!(iter.get(0.into())); - dbg!(iter.get(20.into())); - dbg!(iter.get(21.into())); + dbg!(iter.get(0)); + dbg!(iter.get(20)); + dbg!(iter.get(21)); drop(iter); let reader = vec.create_static_reader(); - dbg!(vec.take(10.into(), &reader)?); - dbg!(vec.get_or_read(10.into(), &reader)?); + dbg!(vec.take(10, &reader)?); + dbg!(vec.get_or_read(10, &reader)?); dbg!(vec.holes()); drop(reader); @@ -124,17 +125,17 @@ fn main() -> Result<(), Box> { dbg!(vec.holes()); let reader = vec.create_static_reader(); - dbg!(vec.get_or_read(10.into(), &reader)?); + dbg!(vec.get_or_read(10, &reader)?); drop(reader); - vec.update(10.into(), 10)?; - vec.update(0.into(), 10)?; + vec.update(10, 10)?; + vec.update(0, 10)?; let reader = vec.create_static_reader(); dbg!( vec.holes(), - vec.get_or_read(0.into(), &reader)?, - vec.get_or_read(10.into(), &reader)? + vec.get_or_read(0, &reader)?, + vec.get_or_read(10, &reader)? ); drop(reader); diff --git a/crates/brk_core/src/error.rs b/crates/brk_vecs/src/error.rs similarity index 61% rename from crates/brk_core/src/error.rs rename to crates/brk_vecs/src/error.rs index 7a158c7bb..de0c733ff 100644 --- a/crates/brk_core/src/error.rs +++ b/crates/brk_vecs/src/error.rs @@ -11,28 +11,17 @@ pub type Result = result::Result; pub enum Error { IO(io::Error), SerdeJson(serde_json::Error), - Jiff(jiff::Error), - Fjall(fjall::Error), SystemTimeError(time::SystemTimeError), + PCO(pco::errors::PcoError), ZeroCopyError, WrongEndian, DifferentVersion { found: Version, expected: Version }, - UnexpectedData, - MmapsVecIsTooSmall, IndexTooHigh, - EmptyVec, - IndexTooLow, - ExpectFileToHaveIndex, ExpectVecToHaveIndex, FailedKeyTryIntoUsize, - UnsupportedUnflushedState, - RangeFromAfterTo(usize, usize), DifferentCompressionMode, WrongLength, - WrongAddressType, - UnindexableDate, - Str(&'static str), String(String), } @@ -49,15 +38,9 @@ impl From for Error { } } -impl From for Error { - fn from(value: jiff::Error) -> Self { - Self::Jiff(value) - } -} - -impl From for Error { - fn from(value: fjall::Error) -> Self { - Self::Fjall(value) +impl From for Error { + fn from(value: pco::errors::PcoError) -> Self { + Self::PCO(value) } } @@ -83,12 +66,10 @@ impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Error::IO(error) => Debug::fmt(&error, f), + Error::PCO(error) => Debug::fmt(&error, f), Error::SystemTimeError(error) => Debug::fmt(&error, f), Error::SerdeJson(error) => Debug::fmt(&error, f), - Error::Jiff(error) => Debug::fmt(&error, f), - Error::Fjall(error) => Debug::fmt(&error, f), Error::ZeroCopyError => write!(f, "ZeroCopy error"), - Error::UnexpectedData => write!(f, "Unexpected data"), Error::WrongEndian => write!(f, "Wrong endian"), Error::DifferentVersion { found, expected } => { @@ -97,28 +78,11 @@ impl fmt::Display for Error { "Different version found: {found:?}, expected: {expected:?}" ) } - Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"), Error::IndexTooHigh => write!(f, "Index too high"), - Error::IndexTooLow => write!(f, "Index too low"), - Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"), Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"), Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"), - Error::UnsupportedUnflushedState => { - write!( - f, - "Unsupported unflush state, please flush before using this function" - ) - } - Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"), Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"), - Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"), Error::WrongLength => write!(f, "Wrong length"), - Error::WrongAddressType => write!(f, "Wrong address type"), - Error::UnindexableDate => write!( - f, - "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater" - ), - Error::Str(s) => write!(f, "{s}"), Error::String(s) => write!(f, "{s}"), } diff --git a/crates/brk_exit/src/lib.rs b/crates/brk_vecs/src/exit.rs similarity index 71% rename from crates/brk_exit/src/lib.rs rename to crates/brk_vecs/src/exit.rs index 1776e20a0..96420c4fa 100644 --- a/crates/brk_exit/src/lib.rs +++ b/crates/brk_vecs/src/exit.rs @@ -1,8 +1,3 @@ -#![doc = include_str!("../README.md")] -#![doc = "\n## Example\n\n```rust"] -#![doc = include_str!("../examples/main.rs")] -#![doc = "```"] - use std::{process::exit, sync::Arc}; use log::info; @@ -13,9 +8,11 @@ pub struct Exit(Arc>); impl Exit { pub fn new() -> Self { - let arc = Arc::new(RwLock::new(())); + Self(Arc::new(RwLock::new(()))) + } - let copy = arc.clone(); + pub fn set_ctrlc_handler(&self) { + let copy = self.0.clone(); ctrlc::set_handler(move || { if copy.is_locked() { @@ -26,8 +23,6 @@ impl Exit { exit(0); }) .expect("Error setting Ctrl-C handler"); - - Self(arc) } pub fn lock(&self) -> RwLockReadGuard<'_, ()> { diff --git a/crates/brk_vecs/src/file/layout.rs b/crates/brk_vecs/src/file/layout.rs index 5bef90b0b..c13259835 100644 --- a/crates/brk_vecs/src/file/layout.rs +++ b/crates/brk_vecs/src/file/layout.rs @@ -1,7 +1,6 @@ use std::collections::BTreeMap; -use brk_core::Error; -use brk_core::Result; +use crate::{Error, Result}; use super::{Region, Regions}; diff --git a/crates/brk_vecs/src/file/mod.rs b/crates/brk_vecs/src/file/mod.rs index cce12cc85..001a9de65 100644 --- a/crates/brk_vecs/src/file/mod.rs +++ b/crates/brk_vecs/src/file/mod.rs @@ -5,7 +5,6 @@ use std::{ sync::Arc, }; -use brk_core::{Error, Result}; use libc::off_t; use log::info; use memmap2::{MmapMut, MmapOptions}; @@ -24,6 +23,8 @@ pub use reader::*; pub use region::*; use regions::*; +use crate::{Error, Result}; + pub const PAGE_SIZE: u64 = 4096; pub const PAGE_SIZE_MINUS_1: u64 = PAGE_SIZE - 1; diff --git a/crates/brk_vecs/src/file/reader.rs b/crates/brk_vecs/src/file/reader.rs index ad0df263d..e95bb0bfb 100644 --- a/crates/brk_vecs/src/file/reader.rs +++ b/crates/brk_vecs/src/file/reader.rs @@ -31,4 +31,8 @@ impl<'a> Reader<'a> { pub fn region(&self) -> &Region { &self.region } + + pub fn prefixed(&self, offset: usize) -> &[u8] { + &self.mmap[offset..] + } } diff --git a/crates/brk_vecs/src/file/regions.rs b/crates/brk_vecs/src/file/regions.rs index 682089e0b..cd10f160d 100644 --- a/crates/brk_vecs/src/file/regions.rs +++ b/crates/brk_vecs/src/file/regions.rs @@ -6,11 +6,12 @@ use std::{ sync::Arc, }; -use brk_core::{Error, Result}; use memmap2::MmapMut; use parking_lot::{RwLock, RwLockWriteGuard}; use zerocopy::{FromBytes, IntoBytes}; +use crate::{Error, Result}; + use super::{ Identifier, PAGE_SIZE, region::{Region, SIZE_OF_REGION}, diff --git a/crates/brk_vecs/src/lib.rs b/crates/brk_vecs/src/lib.rs index 27363e3de..bc49cc1cd 100644 --- a/crates/brk_vecs/src/lib.rs +++ b/crates/brk_vecs/src/lib.rs @@ -1,13 +1,32 @@ +#![doc = include_str!("../README.md")] +#![doc = "\n## Examples\n"] +#![doc = "\n### File\n\n```rust"] +#![doc = include_str!("../examples/file.rs")] +#![doc = "```"] +#![doc = "\n### Raw\n\n```rust"] +#![doc = include_str!("../examples/raw.rs")] +#![doc = "```"] + +mod error; +mod exit; mod file; +mod stamp; mod traits; mod variants; +mod version; + +pub use brk_vecs_macros::StoredCompressed; +pub use pco::data_types::LatentType; use variants::*; +pub use error::*; +pub use exit::*; pub use file::{File, PAGE_SIZE, Reader}; +pub use stamp::*; pub use traits::*; pub use variants::{ - AnyStampedVec, CompressedVec, Computation, ComputedVec, ComputedVecFrom1, ComputedVecFrom2, - ComputedVecFrom3, EagerVec, Format, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, RawVec, Stamp, - StampedVec, StoredVec, + CompressedVec, Computation, ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, + EagerVec, Format, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, RawVec, StoredVec, }; +pub use version::*; diff --git a/crates/brk_vecs/src/variants/stamped/stamp.rs b/crates/brk_vecs/src/stamp.rs similarity index 100% rename from crates/brk_vecs/src/variants/stamped/stamp.rs rename to crates/brk_vecs/src/stamp.rs diff --git a/crates/brk_vecs/src/traits/any.rs b/crates/brk_vecs/src/traits/any.rs index 84c516788..7c8498c64 100644 --- a/crates/brk_vecs/src/traits/any.rs +++ b/crates/brk_vecs/src/traits/any.rs @@ -1,6 +1,4 @@ -use brk_core::{Height, Version}; - -use super::{BoxedVecIterator, StoredIndex, StoredType}; +use crate::{Stamp, Version}; pub fn i64_to_usize(i: i64, len: usize) -> usize { if i >= 0 { @@ -20,7 +18,7 @@ pub trait AnyVec: Send + Sync { } fn index_type_to_string(&self) -> &'static str; fn value_type_to_size_of(&self) -> usize; - fn etag(&self, height: Height, to: Option) -> String { + fn etag(&self, stamp: Stamp, to: Option) -> String { let len = self.len(); format!( "{}-{}-{}", @@ -33,7 +31,7 @@ pub trait AnyVec: Send + Sync { } }), u64::from(self.version()), - u32::from(height), + u64::from(stamp), ) } @@ -43,60 +41,3 @@ pub trait AnyVec: Send + Sync { i64_to_usize(i, len) } } - -pub trait AnyIterableVec: AnyVec { - #[allow(clippy::wrong_self_convention)] - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredType + 'a; - - fn iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredType + 'a, - { - self.boxed_iter() - } - - fn iter_at<'a>(&'a self, i: I) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredType + 'a, - { - let mut iter = self.boxed_iter(); - iter.set(i); - iter - } - - fn iter_at_<'a>(&'a self, i: usize) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredType + 'a, - { - let mut iter = self.boxed_iter(); - iter.set_(i); - iter - } -} - -pub trait CloneableAnyIterableVec: AnyIterableVec { - fn boxed_clone(&self) -> Box>; -} - -impl CloneableAnyIterableVec for U -where - U: 'static + AnyIterableVec + Clone, -{ - fn boxed_clone(&self) -> Box> { - Box::new(self.clone()) - } -} - -impl Clone for Box> { - fn clone(&self) -> Self { - self.boxed_clone() - } -} - -pub type BoxedAnyIterableVec = Box>; diff --git a/crates/brk_core/src/utils/checked_sub.rs b/crates/brk_vecs/src/traits/checked_sub.rs similarity index 100% rename from crates/brk_core/src/utils/checked_sub.rs rename to crates/brk_vecs/src/traits/checked_sub.rs diff --git a/crates/brk_vecs/src/traits/collectable.rs b/crates/brk_vecs/src/traits/collectable.rs index 87d7b840b..bdde9acdf 100644 --- a/crates/brk_vecs/src/traits/collectable.rs +++ b/crates/brk_vecs/src/traits/collectable.rs @@ -1,14 +1,12 @@ -use brk_core::{Error, Result}; +use crate::{Error, Result, i64_to_usize}; -use crate::i64_to_usize; - -use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType}; +use super::{AnyIterableVec, AnyVec, StoredIndex, StoredRaw}; pub trait CollectableVec: AnyVec + AnyIterableVec where Self: Clone, I: StoredIndex, - T: StoredType, + T: StoredRaw, { fn collect(&self) -> Result> { self.collect_range(None, None) @@ -63,7 +61,7 @@ impl CollectableVec for V where V: AnyVec + AnyIterableVec + Clone, I: StoredIndex, - T: StoredType, + T: StoredRaw, { } diff --git a/crates/brk_vecs/src/traits/compressed.rs b/crates/brk_vecs/src/traits/compressed.rs new file mode 100644 index 000000000..4993c3a9f --- /dev/null +++ b/crates/brk_vecs/src/traits/compressed.rs @@ -0,0 +1,98 @@ +use pco::data_types::Number; + +use super::StoredRaw; + +pub trait TransparentStoredCompressed {} + +pub trait StoredCompressed +where + Self: StoredRaw + Copy + 'static + TransparentStoredCompressed, +{ + type NumberType: pco::data_types::Number; +} + +pub trait AsInnerSlice +where + T: Number, +{ + fn as_inner_slice(&self) -> &[T]; +} + +impl AsInnerSlice for [T] +where + T: StoredCompressed, +{ + fn as_inner_slice(&self) -> &[T::NumberType] { + assert_eq!( + std::mem::size_of::(), + std::mem::size_of::() + ); + assert_eq!( + std::mem::align_of::(), + std::mem::align_of::() + ); + unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T::NumberType, self.len()) } + } +} + +pub trait FromInnerSlice { + fn from_inner_slice(slice: Vec) -> Vec + where + Self: Sized; +} + +impl FromInnerSlice for T +where + T: StoredCompressed, +{ + fn from_inner_slice(vec: Vec) -> Vec { + assert_eq!( + std::mem::size_of::(), + std::mem::size_of::() + ); + assert_eq!( + std::mem::align_of::(), + std::mem::align_of::() + ); + + let mut vec = std::mem::ManuallyDrop::new(vec); + unsafe { Vec::from_raw_parts(vec.as_mut_ptr() as *mut T, vec.len(), vec.capacity()) } + } +} + +impl TransparentStoredCompressed for u16 {} +impl StoredCompressed for u16 { + type NumberType = u16; +} +impl TransparentStoredCompressed for u32 {} +impl StoredCompressed for u32 { + type NumberType = u32; +} +impl TransparentStoredCompressed for u64 {} +impl StoredCompressed for u64 { + type NumberType = u64; +} +impl TransparentStoredCompressed for i16 {} +impl StoredCompressed for i16 { + type NumberType = i16; +} +impl TransparentStoredCompressed for i32 {} +impl StoredCompressed for i32 { + type NumberType = i32; +} +impl TransparentStoredCompressed for i64 {} +impl StoredCompressed for i64 { + type NumberType = i64; +} +impl TransparentStoredCompressed for f32 {} +impl StoredCompressed for f32 { + type NumberType = f32; +} +impl TransparentStoredCompressed for f64 {} +impl StoredCompressed for f64 { + type NumberType = f64; +} +impl TransparentStoredCompressed for () {} +impl StoredCompressed for () { + type NumberType = u16; +} diff --git a/crates/brk_core/src/traits/from_coarser.rs b/crates/brk_vecs/src/traits/from_coarser.rs similarity index 100% rename from crates/brk_core/src/traits/from_coarser.rs rename to crates/brk_vecs/src/traits/from_coarser.rs diff --git a/crates/brk_vecs/src/traits/generic.rs b/crates/brk_vecs/src/traits/generic.rs index a3bd75748..65f95ecd1 100644 --- a/crates/brk_vecs/src/traits/generic.rs +++ b/crates/brk_vecs/src/traits/generic.rs @@ -4,24 +4,22 @@ use std::{ collections::{BTreeMap, BTreeSet}, }; -use brk_core::{Error, Result}; +use crate::{AnyStoredVec, Error, Exit, HEADER_OFFSET, Result, Stamp, file::Reader}; -use crate::{AnyVec, File, HEADER_OFFSET, Header, file::Reader}; +const ONE_KIB: usize = 1024; +const ONE_MIB: usize = ONE_KIB * ONE_KIB; +const MAX_CACHE_SIZE: usize = 256 * ONE_MIB; -use super::{StoredIndex, StoredType}; +use super::{StoredIndex, StoredRaw}; pub trait GenericStoredVec: Send + Sync where - Self: AnyVec, + Self: AnyStoredVec, I: StoredIndex, - T: StoredType, + T: StoredRaw, { const SIZE_OF_T: usize = size_of::(); - fn file(&self) -> &File; - - fn region_index(&self) -> usize; - /// Be careful with deadlocks /// /// You'll want to drop the reader before mutable ops @@ -44,13 +42,13 @@ where #[inline] fn unwrap_read(&self, index: I, reader: &Reader) -> T { - self.read(index, reader).unwrap().unwrap() + self.read(index, reader).unwrap() } #[inline] - fn read(&self, index: I, reader: &Reader) -> Result> { + fn read(&self, index: I, reader: &Reader) -> Result { self.read_(index.to_usize()?, reader) } - fn read_(&self, index: usize, reader: &Reader) -> Result>; + fn read_(&self, index: usize, reader: &Reader) -> Result; #[inline] fn get_or_read(&self, index: I, reader: &Reader) -> Result>> { @@ -81,7 +79,7 @@ where return Ok(None); } - Ok(self.read_(index, reader)?.map(Cow::Owned)) + Ok(Some(Cow::Owned(self.read_(index, reader)?))) } #[inline] @@ -89,8 +87,6 @@ where self.stored_len() + self.pushed_len() } - fn stored_len(&self) -> usize; - fn pushed(&self) -> &[T]; #[inline] fn pushed_len(&self) -> usize { @@ -102,6 +98,47 @@ where self.mut_pushed().push(value) } + #[inline] + fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { + let len = self.len(); + match len.cmp(&index.to_usize()?) { + Ordering::Greater => { + // dbg!(len, index, &self.pathbuf); + // panic!(); + Ok(()) + } + Ordering::Equal => { + self.push(value); + Ok(()) + } + Ordering::Less => { + dbg!(index, value, len, self.header()); + Err(Error::IndexTooHigh) + } + } + } + + #[inline] + fn forced_push_at(&mut self, index: I, value: T, exit: &Exit) -> Result<()> { + match self.len().cmp(&index.to_usize()?) { + Ordering::Less => { + return Err(Error::IndexTooHigh); + } + ord => { + if ord == Ordering::Greater { + self.safe_truncate_if_needed(index, exit)?; + } + self.push(value); + } + } + + if self.pushed_len() * Self::SIZE_OF_T >= MAX_CACHE_SIZE { + self.safe_flush(exit) + } else { + Ok(()) + } + } + #[inline] fn update_or_push(&mut self, index: I, value: T) -> Result<()> { let len = self.len(); @@ -191,9 +228,6 @@ where Ok(()) } - fn header(&self) -> &Header; - fn mut_header(&mut self) -> &mut Header; - fn reset(&mut self) -> Result<()>; #[inline] @@ -217,10 +251,19 @@ where index < self.len_() } - fn flush(&mut self) -> Result<()>; - fn truncate_if_needed(&mut self, index: I) -> Result<()>; + fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> { + let _lock = exit.lock(); + self.truncate_if_needed(index) + } + + #[inline] + fn truncate_if_needed_with_stamp(&mut self, index: I, stamp: Stamp) -> Result<()> { + self.update_stamp(stamp); + self.truncate_if_needed(index) + } + fn index_to_name(&self) -> String { format!("{}_to_{}", I::to_string(), self.name()) } diff --git a/crates/brk_vecs/src/traits/index.rs b/crates/brk_vecs/src/traits/index.rs index 723d28b45..7d6c00955 100644 --- a/crates/brk_vecs/src/traits/index.rs +++ b/crates/brk_vecs/src/traits/index.rs @@ -1,8 +1,9 @@ use std::{fmt::Debug, ops::Add}; -use brk_core::{Error, Printable, Result}; use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; +use crate::{Error, Printable, Result}; + pub trait StoredIndex where Self: Debug @@ -13,8 +14,8 @@ where + Eq + PartialOrd + Ord - + TryInto + From + + TryInto + Add + TryFromBytes + IntoBytes @@ -39,8 +40,8 @@ where + Eq + PartialOrd + Ord - + TryInto + From + + TryInto + Add + TryFromBytes + IntoBytes diff --git a/crates/brk_vecs/src/traits/iterable.rs b/crates/brk_vecs/src/traits/iterable.rs new file mode 100644 index 000000000..b04555b03 --- /dev/null +++ b/crates/brk_vecs/src/traits/iterable.rs @@ -0,0 +1,63 @@ +use crate::{AnyStoredVec, AnyVec, BoxedVecIterator, StoredIndex, StoredRaw}; + +pub trait AnyIterableVec: AnyVec { + #[allow(clippy::wrong_self_convention)] + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredRaw + 'a; + + fn iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredRaw + 'a, + { + self.boxed_iter() + } + + fn iter_at<'a>(&'a self, i: I) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredRaw + 'a, + { + let mut iter = self.boxed_iter(); + iter.set(i); + iter + } + + fn iter_at_<'a>(&'a self, i: usize) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredRaw + 'a, + { + let mut iter = self.boxed_iter(); + iter.set_(i); + iter + } +} + +pub trait AnyStoredIterableVec: AnyIterableVec + AnyStoredVec {} + +impl AnyStoredIterableVec for U where U: 'static + AnyIterableVec + AnyStoredVec +{} + +pub trait AnyCloneableIterableVec: AnyIterableVec { + fn boxed_clone(&self) -> Box>; +} + +impl AnyCloneableIterableVec for U +where + U: 'static + AnyIterableVec + Clone, +{ + fn boxed_clone(&self) -> Box> { + Box::new(self.clone()) + } +} + +impl Clone for Box> { + fn clone(&self) -> Self { + self.boxed_clone() + } +} + +pub type AnyBoxedIterableVec = Box>; diff --git a/crates/brk_vecs/src/traits/iterator.rs b/crates/brk_vecs/src/traits/iterator.rs index 097e513a7..a33ae2845 100644 --- a/crates/brk_vecs/src/traits/iterator.rs +++ b/crates/brk_vecs/src/traits/iterator.rs @@ -1,8 +1,6 @@ use std::{borrow::Cow, iter::Skip}; -use brk_core::Printable; - -use super::{StoredIndex, StoredType}; +use super::{Printable, StoredIndex, StoredRaw}; pub trait BaseVecIterator: Iterator { fn mut_index(&mut self) -> &mut usize; @@ -36,7 +34,7 @@ pub trait BaseVecIterator: Iterator { pub trait VecIterator<'a>: BaseVecIterator)> { type I: StoredIndex; - type T: StoredType + 'a; + type T: StoredRaw + 'a; #[inline] fn set(&mut self, i: Self::I) { @@ -62,7 +60,7 @@ pub trait VecIterator<'a>: BaseVecIterator)> { fn unwrap_get_inner_(&mut self, i: usize) -> Self::T { self.get_(i) .unwrap_or_else(|| { - dbg!(self.name(), i, self.len(), Self::I::to_string()); + dbg!(self.name(), i, self.len(), std::any::type_name::()); panic!("unwrap_get_inner_") }) .into_owned() @@ -95,7 +93,7 @@ impl<'a, I, T, Iter> VecIterator<'a> for Iter where Iter: BaseVecIterator)>, I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, { type I = I; type T = T; diff --git a/crates/brk_vecs/src/traits/mod.rs b/crates/brk_vecs/src/traits/mod.rs index d07c2a598..900aef8bf 100644 --- a/crates/brk_vecs/src/traits/mod.rs +++ b/crates/brk_vecs/src/traits/mod.rs @@ -1,13 +1,25 @@ mod any; +mod checked_sub; mod collectable; +mod compressed; +mod from_coarser; mod generic; mod index; +mod iterable; mod iterator; -mod r#type; +mod printable; +mod raw; +mod stored; pub use any::*; +pub use checked_sub::*; pub use collectable::*; +pub use compressed::*; +pub use from_coarser::*; pub use generic::*; pub use index::*; +pub use iterable::*; pub use iterator::*; -pub use r#type::*; +pub use printable::*; +pub use raw::*; +pub use stored::*; diff --git a/crates/brk_vecs/src/traits/printable.rs b/crates/brk_vecs/src/traits/printable.rs new file mode 100644 index 000000000..b2f405821 --- /dev/null +++ b/crates/brk_vecs/src/traits/printable.rs @@ -0,0 +1,14 @@ +pub trait Printable { + fn to_string() -> &'static str; + fn to_possible_strings() -> &'static [&'static str]; +} + +impl Printable for usize { + fn to_string() -> &'static str { + "usize" + } + + fn to_possible_strings() -> &'static [&'static str] { + &["usize"] + } +} diff --git a/crates/brk_vecs/src/traits/type.rs b/crates/brk_vecs/src/traits/raw.rs similarity index 69% rename from crates/brk_vecs/src/traits/type.rs rename to crates/brk_vecs/src/traits/raw.rs index 3acb2f7bf..f2f9447a4 100644 --- a/crates/brk_vecs/src/traits/type.rs +++ b/crates/brk_vecs/src/traits/raw.rs @@ -1,14 +1,14 @@ use std::fmt::Debug; use serde::Serialize; -use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -pub trait StoredType +pub trait StoredRaw where Self: Sized + Debug + Clone - + TryFromBytes + + FromBytes + IntoBytes + Immutable + KnownLayout @@ -18,11 +18,11 @@ where { } -impl StoredType for T where +impl StoredRaw for T where T: Sized + Debug + Clone - + TryFromBytes + + FromBytes + IntoBytes + Immutable + KnownLayout diff --git a/crates/brk_vecs/src/traits/stored.rs b/crates/brk_vecs/src/traits/stored.rs new file mode 100644 index 000000000..e21660308 --- /dev/null +++ b/crates/brk_vecs/src/traits/stored.rs @@ -0,0 +1,35 @@ +use crate::{AnyVec, Exit, File, Result, Stamp, variants::Header}; + +pub trait AnyStoredVec: AnyVec { + fn file(&self) -> &File; + + fn region_index(&self) -> usize; + + fn header(&self) -> &Header; + + fn mut_header(&mut self) -> &mut Header; + + fn flush(&mut self) -> Result<()>; + + #[inline] + fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + let _lock = exit.lock(); + self.flush() + } + + fn stored_len(&self) -> usize; + + fn update_stamp(&mut self, stamp: Stamp) { + self.mut_header().update_stamp(stamp); + } + + fn stamp(&self) -> Stamp { + self.header().stamp() + } + + #[inline] + fn stamped_flush(&mut self, stamp: Stamp) -> Result<()> { + self.update_stamp(stamp); + self.flush() + } +} diff --git a/crates/brk_vecs/src/variants/compressed/compressed_pages_meta.rs b/crates/brk_vecs/src/variants/compressed/compressed_pages_meta.rs index 6d170e064..941def5e7 100644 --- a/crates/brk_vecs/src/variants/compressed/compressed_pages_meta.rs +++ b/crates/brk_vecs/src/variants/compressed/compressed_pages_meta.rs @@ -4,10 +4,11 @@ use std::{ path::{Path, PathBuf}, }; -use brk_core::Result; use rayon::prelude::*; use zerocopy::{IntoBytes, TryFromBytes}; +use crate::Result; + use super::{CompressedPageMetadata, UnsafeSlice}; #[derive(Debug, Clone)] diff --git a/crates/brk_vecs/src/variants/compressed/mod.rs b/crates/brk_vecs/src/variants/compressed/mod.rs index 5e0e590c1..32af408c0 100644 --- a/crates/brk_vecs/src/variants/compressed/mod.rs +++ b/crates/brk_vecs/src/variants/compressed/mod.rs @@ -5,16 +5,15 @@ use std::{ sync::Arc, }; -use brk_core::{Error, Result, Version}; use memmap2::Mmap; use parking_lot::{RwLock, RwLockReadGuard}; +use pco::data_types::Number; use rayon::prelude::*; -use zstd::DEFAULT_COMPRESSION_LEVEL; use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec, - File, GenericStoredVec, HEADER_OFFSET, Header, RawVec, Reader, StoredIndex, StoredType, - UnsafeSlice, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, AsInnerSlice, BaseVecIterator, + BoxedVecIterator, CollectableVec, Error, File, FromInnerSlice, GenericStoredVec, HEADER_OFFSET, + Header, RawVec, Reader, Result, StoredCompressed, StoredIndex, UnsafeSlice, Version, }; mod compressed_page_meta; @@ -24,9 +23,8 @@ use compressed_page_meta::*; use compressed_pages_meta::*; const ONE_KIB: usize = 1024; -const ONE_MIB: usize = ONE_KIB * ONE_KIB; -pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB; -pub const MAX_PAGE_SIZE: usize = 64 * ONE_KIB; +const MAX_PAGE_SIZE: usize = 16 * ONE_KIB; +const PCO_COMPRESSION_LEVEL: usize = 4; const VERSION: Version = Version::TWO; @@ -39,11 +37,10 @@ pub struct CompressedVec { impl CompressedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T; - pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE; /// Same as import but will reset the vec under certain errors, so be careful ! pub fn forced_import(file: &Arc, name: &str, mut version: Version) -> Result { @@ -117,15 +114,12 @@ where let len = page.bytes_len as usize; let offset = page.start as usize; - let slice = reader.read(offset as u64, (offset + len) as u64); + let slice: &[u8] = reader.read(offset as u64, (offset + len) as u64); - Ok(zstd::decode_all(slice) - .inspect_err(|_| { - dbg!((len, offset, page_index, slice)); - })? - .chunks(Self::SIZE_OF_T) - .map(|slice| T::try_read_from_bytes(slice).unwrap()) - .collect::>()) + let vec: Vec = pco::standalone::simple_decompress(slice)?; + let vec: Vec = T::from_inner_slice(vec); + + Ok(vec) } fn compress_page(chunk: &[T]) -> Vec { @@ -133,16 +127,7 @@ where panic!(); } - let mut bytes: Vec = vec![0; chunk.len() * Self::SIZE_OF_T]; - - let unsafe_bytes = UnsafeSlice::new(&mut bytes); - - chunk - .into_par_iter() - .enumerate() - .for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes())); - - zstd::encode_all(bytes.as_slice(), DEFAULT_COMPRESSION_LEVEL).unwrap() + pco::standalone::simpler_compress(chunk.as_inner_slice(), PCO_COMPRESSION_LEVEL).unwrap() } #[inline] @@ -173,10 +158,50 @@ where } } -impl GenericStoredVec for CompressedVec +impl Clone for CompressedVec { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + pages_meta: self.pages_meta.clone(), + } + } +} + +impl AnyVec for CompressedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, +{ + #[inline] + fn version(&self) -> Version { + self.inner.version() + } + + #[inline] + fn name(&self) -> &str { + self.inner.name() + } + + #[inline] + fn len(&self) -> usize { + self.len_() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } +} + +impl AnyStoredVec for CompressedVec +where + I: StoredIndex, + T: StoredCompressed, { fn file(&self) -> &File { self.inner.file() @@ -186,17 +211,6 @@ where self.inner.region_index() } - #[inline] - fn read_(&self, index: usize, reader: &Reader) -> Result> { - let page_index = Self::index_to_page_index(index); - let decoded_index = index % Self::PER_PAGE; - - Ok(self - .decode_page(page_index, reader)? - .get(decoded_index) - .cloned()) - } - fn header(&self) -> &Header { self.inner.header() } @@ -210,31 +224,6 @@ where self.inner.stored_len() } - #[inline] - fn pushed(&self) -> &[T] { - self.inner.pushed() - } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - self.inner.mut_pushed() - } - #[inline] - fn holes(&self) -> &BTreeSet { - self.inner.holes() - } - #[inline] - fn mut_holes(&mut self) -> &mut BTreeSet { - panic!("unsupported") - } - #[inline] - fn updated(&self) -> &BTreeMap { - self.inner.updated() - } - #[inline] - fn mut_updated(&mut self) -> &mut BTreeMap { - panic!("unsupported") - } - fn flush(&mut self) -> Result<()> { todo!(); @@ -321,6 +310,49 @@ where Ok(()) } +} + +impl GenericStoredVec for CompressedVec +where + I: StoredIndex, + T: StoredCompressed, +{ + #[inline] + fn read_(&self, index: usize, reader: &Reader) -> Result { + let page_index = Self::index_to_page_index(index); + let decoded_index = index % Self::PER_PAGE; + + Ok(unsafe { + self.decode_page(page_index, reader)? + .get_unchecked(decoded_index) + .clone() + }) + } + + #[inline] + fn pushed(&self) -> &[T] { + self.inner.pushed() + } + #[inline] + fn mut_pushed(&mut self) -> &mut Vec { + self.inner.mut_pushed() + } + #[inline] + fn holes(&self) -> &BTreeSet { + self.inner.holes() + } + #[inline] + fn mut_holes(&mut self) -> &mut BTreeSet { + panic!("unsupported") + } + #[inline] + fn updated(&self) -> &BTreeMap { + self.inner.updated() + } + #[inline] + fn mut_updated(&mut self) -> &mut BTreeMap { + panic!("unsupported") + } fn reset(&mut self) -> Result<()> { // let mut pages_meta = (**self.pages_meta.load()).clone(); @@ -377,46 +409,6 @@ where } } -impl AnyVec for CompressedVec -where - I: StoredIndex, - T: StoredType, -{ - #[inline] - fn version(&self) -> Version { - self.inner.version() - } - - #[inline] - fn name(&self) -> &str { - self.inner.name() - } - - #[inline] - fn len(&self) -> usize { - self.len_() - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl Clone for CompressedVec { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - pages_meta: self.pages_meta.clone(), - } - } -} - #[derive(Debug)] pub struct CompressedVecIterator<'a, I, T> { vec: &'a CompressedVec, @@ -430,7 +422,7 @@ pub struct CompressedVecIterator<'a, I, T> { impl CompressedVecIterator<'_, I, T> where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { const SIZE_OF_T: usize = size_of::(); const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; @@ -439,7 +431,7 @@ where impl BaseVecIterator for CompressedVecIterator<'_, I, T> where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { #[inline] fn mut_index(&mut self) -> &mut usize { @@ -460,7 +452,7 @@ where impl<'a, I, T> Iterator for CompressedVecIterator<'a, I, T> where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { type Item = (I, Cow<'a, T>); @@ -508,7 +500,7 @@ where impl<'a, I, T> IntoIterator for &'a CompressedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { type Item = (I, Cow<'a, T>); type IntoIter = CompressedVecIterator<'a, I, T>; @@ -531,7 +523,7 @@ where impl AnyIterableVec for CompressedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -544,7 +536,7 @@ where impl AnyCollectableVec for CompressedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/computed/computation.rs b/crates/brk_vecs/src/variants/computed/computation.rs index cb9c491d1..0f6142e61 100644 --- a/crates/brk_vecs/src/variants/computed/computation.rs +++ b/crates/brk_vecs/src/variants/computed/computation.rs @@ -1,9 +1,6 @@ -use clap_derive::ValueEnum; use serde::{Deserialize, Serialize}; -#[derive( - Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum, -)] +#[derive(Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize)] pub enum Computation { Eager, #[default] diff --git a/crates/brk_vecs/src/variants/computed/mod.rs b/crates/brk_vecs/src/variants/computed/mod.rs index 792fda5f0..9b35d2665 100644 --- a/crates/brk_vecs/src/variants/computed/mod.rs +++ b/crates/brk_vecs/src/variants/computed/mod.rs @@ -1,11 +1,9 @@ use std::{borrow::Cow, sync::Arc}; -use brk_core::{Result, StoredPhantom, Version}; -use brk_exit::Exit; - use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, - BoxedVecIterator, CollectableVec, File, Format, StoredIndex, StoredType, + AnyBoxedIterableVec, AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, + BoxedVecIterator, CollectableVec, Exit, File, Format, Result, StoredCompressed, StoredIndex, + StoredRaw, Version, }; use super::{ @@ -24,25 +22,24 @@ where S2T: Clone, S3T: Clone, { - From1(BoxedAnyIterableVec, ComputeFrom1), + From1(AnyBoxedIterableVec, ComputeFrom1), From2( - (BoxedAnyIterableVec, BoxedAnyIterableVec), + (AnyBoxedIterableVec, AnyBoxedIterableVec), ComputeFrom2, ), From3( ( - BoxedAnyIterableVec, - BoxedAnyIterableVec, - BoxedAnyIterableVec, + AnyBoxedIterableVec, + AnyBoxedIterableVec, + AnyBoxedIterableVec, ), ComputeFrom3, ), } -pub type ComputedVecFrom1 = - ComputedVec; +pub type ComputedVecFrom1 = ComputedVec; pub type ComputedVecFrom2 = - ComputedVec; + ComputedVec; pub type ComputedVecFrom3 = ComputedVec; @@ -65,13 +62,13 @@ where impl ComputedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { pub fn forced_import_or_init_from_1( computation: Computation, @@ -79,7 +76,7 @@ where name: &str, version: Version, format: Format, - source: BoxedAnyIterableVec, + source: AnyBoxedIterableVec, compute: ComputeFrom1, ) -> Result { Ok(match computation { @@ -100,8 +97,8 @@ where name: &str, version: Version, format: Format, - source1: BoxedAnyIterableVec, - source2: BoxedAnyIterableVec, + source1: AnyBoxedIterableVec, + source2: AnyBoxedIterableVec, compute: ComputeFrom2, ) -> Result { Ok(match computation { @@ -122,9 +119,9 @@ where name: &str, version: Version, format: Format, - source1: BoxedAnyIterableVec, - source2: BoxedAnyIterableVec, - source3: BoxedAnyIterableVec, + source1: AnyBoxedIterableVec, + source2: AnyBoxedIterableVec, + source3: AnyBoxedIterableVec, compute: ComputeFrom3, ) -> Result { Ok(match computation { @@ -193,13 +190,13 @@ where impl AnyVec for ComputedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { fn version(&self) -> Version { match self { @@ -254,13 +251,13 @@ impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator for ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> where I: StoredIndex, - T: StoredType, + T: StoredCompressed, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { @@ -277,13 +274,13 @@ impl BaseVecIterator for ComputedVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> where I: StoredIndex, - T: StoredType, + T: StoredCompressed, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { #[inline] fn mut_index(&mut self) -> &mut usize { @@ -319,13 +316,13 @@ impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator for &'a ComputedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { type Item = (I, Cow<'a, T>); type IntoIter = ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; @@ -344,13 +341,13 @@ impl AnyIterableVec for ComputedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -364,13 +361,13 @@ impl AnyCollectableVec for ComputedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/eager.rs b/crates/brk_vecs/src/variants/eager.rs index 560d9a717..015474ae9 100644 --- a/crates/brk_vecs/src/variants/eager.rs +++ b/crates/brk_vecs/src/variants/eager.rs @@ -1,41 +1,29 @@ use core::error; use std::{ borrow::Cow, - cmp::Ordering, + collections::{BTreeMap, BTreeSet}, f32, fmt::Debug, - ops::{Add, Div, Mul}, + ops::{Add, Div, Mul, Sub}, sync::Arc, }; -use brk_core::{ - Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Error, Result, Sats, StoredF32, - StoredUsize, Version, -}; -use brk_exit::Exit; use log::info; use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, File, Format, - GenericStoredVec, Reader, StoredIndex, StoredType, StoredVec, StoredVecIterator, VecIterator, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, BoxedVecIterator, CheckedSub, + CollectableVec, Exit, File, Format, GenericStoredVec, Reader, Result, StoredCompressed, + StoredIndex, StoredRaw, StoredVec, StoredVecIterator, VecIterator, Version, variants::Header, }; -const ONE_KIB: usize = 1024; -const ONE_MIB: usize = ONE_KIB * ONE_KIB; -const MAX_CACHE_SIZE: usize = 256 * ONE_MIB; -const DCA_AMOUNT: Dollars = Dollars::mint(100.0); - #[derive(Debug, Clone)] pub struct EagerVec(StoredVec); -// computed_version: Arc>>, impl EagerVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { - const SIZE_OF: usize = size_of::(); - pub fn forced_import( file: &Arc, value_name: &str, @@ -47,58 +35,27 @@ where )?)) } - fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> { - let _lock = exit.lock(); - self.0.truncate_if_needed(index)?; - Ok(()) - } - #[inline] - pub fn forced_push_at(&mut self, index: I, value: T, exit: &Exit) -> Result<()> { - match self.len().cmp(&index.to_usize()?) { - Ordering::Less => { - return Err(Error::IndexTooHigh); - } - ord => { - if ord == Ordering::Greater { - self.safe_truncate_if_needed(index, exit)?; - } - self.0.push(value); - } - } - - if self.0.pushed_len() * Self::SIZE_OF >= MAX_CACHE_SIZE { - self.safe_flush(exit) - } else { - Ok(()) - } - } - - pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { - let _lock = exit.lock(); - self.0.flush()?; - Ok(()) - } - - pub fn get_or_read(&self, index: I, reader: &Reader) -> Result>> { + pub fn get_or_read<'a, 'b>(&'a self, index: I, reader: &'b Reader) -> Result>> + where + 'a: 'b, + { self.0.get_or_read(index, reader) } pub fn inner_version(&self) -> Version { - self.0.version() + self.version() } fn update_computed_version(&mut self, computed_version: Version) { - self.0 - .mut_header() - .update_computed_version(computed_version); + self.mut_header().update_computed_version(computed_version); } pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { - if version != self.0.header().computed_version() { + if version != self.header().computed_version() { self.update_computed_version(version); if !self.is_empty() { - self.0.reset()?; + self.reset()?; } } @@ -124,7 +81,9 @@ where where F: FnMut(I) -> (I, T), { - self.validate_computed_version_or_reset_file(Version::ZERO + self.0.version() + version)?; + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner_version() + version, + )?; let index = max_from.min(I::from(self.len())); (index.to_usize()?..to).try_for_each(|i| { @@ -143,7 +102,7 @@ where exit: &Exit, ) -> Result<()> where - A: StoredType, + A: StoredRaw, F: FnMut(I) -> (I, T), { self.compute_to(max_from, other.len(), other.version(), t, exit) @@ -157,7 +116,7 @@ where ) -> Result<()> where T: From, - T2: StoredType, + T2: StoredRaw, { self.compute_to( max_from, @@ -177,11 +136,11 @@ where ) -> Result<()> where A: StoredIndex, - B: StoredType, + B: StoredRaw, F: FnMut((A, B, &Self)) -> (I, T), { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + other.version(), + Version::ZERO + self.inner_version() + other.version(), )?; let index = max_from.min(A::from(self.len())); @@ -204,7 +163,7 @@ where T: Add, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + added.version() + adder.version(), + Version::ZERO + self.inner_version() + added.version() + adder.version(), )?; let index = max_from.min(I::from(self.len())); @@ -230,7 +189,7 @@ where T: CheckedSub, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + subtracted.version() + subtracter.version(), + Version::ZERO + self.inner_version() + subtracted.version() + subtracter.version(), )?; let index = max_from.min(I::from(self.len())); @@ -256,10 +215,10 @@ where ) -> Result<()> where T: From + Ord, - T2: StoredType, + T2: StoredRaw, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + source.version(), + Version::ZERO + self.inner_version() + source.version(), )?; let index = max_from.min(I::from(self.len())); @@ -275,8 +234,8 @@ where T::from(source.iter().unwrap_get_inner_(0)) }); } - let max = prev.clone().unwrap().max(T::from(v.into_owned())); - prev.replace(max.clone()); + let max = prev.unwrap().max(T::from(v.into_owned())); + prev.replace(max); self.forced_push_at(i, max, exit) })?; @@ -292,13 +251,13 @@ where exit: &Exit, ) -> Result<()> where - T2: StoredType + Mul, - T3: StoredType, - T4: StoredType, + T2: StoredRaw + Mul, + T3: StoredRaw, + T4: StoredRaw, T: From, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + multiplied.version() + multiplier.version(), + Version::ZERO + self.inner_version() + multiplied.version() + multiplier.version(), )?; let index = max_from.min(I::from(self.len())); @@ -321,8 +280,8 @@ where exit: &Exit, ) -> Result<()> where - T2: StoredType + Mul, - T3: StoredType, + T2: StoredRaw + Mul, + T3: StoredRaw, T4: Div + From, T5: CheckedSub, T: From, @@ -338,8 +297,8 @@ where exit: &Exit, ) -> Result<()> where - T2: StoredType + Mul, - T3: StoredType, + T2: StoredRaw + Mul, + T3: StoredRaw, T4: Div + From, T5: CheckedSub, T: From, @@ -355,8 +314,8 @@ where exit: &Exit, ) -> Result<()> where - T2: StoredType + Mul, - T3: StoredType, + T2: StoredRaw + Mul, + T3: StoredRaw, T4: Div + From, T5: CheckedSub, T: From, @@ -374,14 +333,14 @@ where as_difference: bool, ) -> Result<()> where - T2: StoredType + Mul, - T3: StoredType, + T2: StoredRaw + Mul, + T3: StoredRaw, T4: Div + From, T5: CheckedSub, T: From, { self.validate_computed_version_or_reset_file( - Version::ONE + self.0.version() + divided.version() + divider.version(), + Version::ONE + self.inner_version() + divided.version() + divider.version(), )?; let index = max_from.min(I::from(self.len())); @@ -407,36 +366,6 @@ where self.safe_flush(exit) } - pub fn compute_drawdown( - &mut self, - max_from: I, - close: &impl AnyIterableVec>, - ath: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: From, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + ath.version() + close.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut close_iter = close.iter(); - ath.iter_at(index).try_for_each(|(i, ath)| { - let ath = ath.into_owned(); - if ath == Dollars::ZERO { - self.forced_push_at(i, T::from(StoredF32::default()), exit) - } else { - let close = *close_iter.unwrap_get_inner(i); - let drawdown = StoredF32::from((*ath - *close) / *ath * -100.0); - self.forced_push_at(i, T::from(drawdown), exit) - } - })?; - - self.safe_flush(exit) - } - pub fn compute_inverse_more_to_less( &mut self, max_from: T, @@ -444,15 +373,15 @@ where exit: &Exit, ) -> Result<()> where - I: StoredType + StoredIndex, + I: StoredRaw + StoredIndex, T: StoredIndex, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + other.version(), + Version::ZERO + self.inner_version() + other.version(), )?; let index = max_from.min( - VecIterator::last(self.0.into_iter()).map_or_else(T::default, |(_, v)| v.into_owned()), + VecIterator::last(self.into_iter()).map_or_else(T::default, |(_, v)| v.into_owned()), ); let mut prev_i = None; other.iter_at(index).try_for_each(|(v, i)| -> Result<()> { @@ -470,19 +399,24 @@ where self.safe_flush(exit) } - pub fn compute_inverse_less_to_more( + pub fn compute_inverse_less_to_more( &mut self, max_from: T, first_indexes: &impl AnyIterableVec, - indexes_count: &impl AnyIterableVec, + indexes_count: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where - I: StoredType, + I: StoredRaw, T: StoredIndex, + T2: StoredRaw, + usize: From, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + first_indexes.version() + indexes_count.version(), + Version::ZERO + + self.inner_version() + + first_indexes.version() + + indexes_count.version(), )?; let mut indexes_count_iter = indexes_count.iter(); @@ -492,7 +426,7 @@ where .iter_at(index) .try_for_each(|(value, first_index)| { let first_index = (first_index).to_usize()?; - let count = *indexes_count_iter.unwrap_get_inner(value); + let count = usize::from(indexes_count_iter.unwrap_get_inner(value)); (first_index..first_index + count) .try_for_each(|index| self.forced_push_at(I::from(index), value, exit)) })?; @@ -509,7 +443,7 @@ where ) -> Result<()> where T: From, - T2: StoredType + T2: StoredRaw + StoredIndex + Copy + Add @@ -517,7 +451,7 @@ where + TryInto + Default, >::Error: error::Error + 'static, - T3: StoredType, + T3: StoredRaw, { let opt: Option bool>> = None; self.compute_filtered_count_from_indexes_(max_from, first_indexes, other_to_else, opt, exit) @@ -533,7 +467,7 @@ where ) -> Result<()> where T: From, - T2: StoredType + T2: StoredRaw + StoredIndex + Copy + Add @@ -541,7 +475,7 @@ where + TryInto + Default, >::Error: error::Error + 'static, - T3: StoredType, + T3: StoredRaw, F: FnMut(T2) -> bool, { self.compute_filtered_count_from_indexes_( @@ -563,18 +497,21 @@ where ) -> Result<()> where T: From, - T2: StoredType + T2: StoredRaw + StoredIndex + Copy + Add + CheckedSub + TryInto + Default, - T3: StoredType, + T3: StoredRaw, >::Error: error::Error + 'static, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + first_indexes.version() + other_to_else.version(), + Version::ZERO + + self.inner_version() + + first_indexes.version() + + other_to_else.version(), )?; let mut other_iter = first_indexes.iter(); @@ -607,12 +544,15 @@ where exit: &Exit, ) -> Result<()> where - I: StoredType, + I: StoredRaw, T: From, - A: StoredIndex + StoredType, + A: StoredIndex + StoredRaw, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + self_to_other.version() + other_to_self.version(), + Version::ZERO + + self.inner_version() + + self_to_other.version() + + other_to_self.version(), )?; let mut other_to_self_iter = other_to_self.iter(); @@ -628,20 +568,25 @@ where self.safe_flush(exit) } - pub fn compute_sum_from_indexes( + pub fn compute_sum_from_indexes( &mut self, max_from: I, first_indexes: &impl AnyIterableVec, - indexes_count: &impl AnyIterableVec, + indexes_count: &impl AnyIterableVec, source: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where T: From + Add, - T2: StoredIndex + StoredType, + T2: StoredIndex + StoredRaw, + T3: StoredRaw, + usize: From, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + first_indexes.version() + indexes_count.version(), + Version::ZERO + + self.inner_version() + + first_indexes.version() + + indexes_count.version(), )?; let mut indexes_count_iter = indexes_count.iter(); @@ -650,12 +595,12 @@ where first_indexes .iter_at(index) .try_for_each(|(i, first_index)| { - let count = *indexes_count_iter.unwrap_get_inner(i); + let count = usize::from(indexes_count_iter.unwrap_get_inner(i)); let first_index = first_index.unwrap_to_usize(); let range = first_index..first_index + count; let mut sum = T::from(0_usize); range.into_iter().for_each(|i| { - sum = sum.clone() + source_iter.unwrap_get_inner(T2::from(i)); + sum = sum + source_iter.unwrap_get_inner(T2::from(i)); }); self.forced_push_at(i, sum, exit) })?; @@ -673,7 +618,7 @@ where T: From + Add, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + others.iter().map(|v| v.version()).sum(), + Version::ZERO + self.inner_version() + others.iter().map(|v| v.version()).sum(), )?; if others.is_empty() { @@ -690,7 +635,7 @@ where .try_for_each(|(i, v)| { let mut sum = v.into_owned(); others_iter.iter_mut().for_each(|iter| { - sum = sum.clone() + iter.unwrap_get_inner(i); + sum = sum + iter.unwrap_get_inner(i); }); self.forced_push_at(i, sum, exit) })?; @@ -708,7 +653,7 @@ where T: From + Add + Ord, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + others.iter().map(|v| v.version()).sum(), + Version::ZERO + self.inner_version() + others.iter().map(|v| v.version()).sum(), )?; if others.is_empty() { @@ -728,7 +673,7 @@ where .iter_mut() .map(|iter| iter.unwrap_get_inner(i)) .min() - .map_or(min.clone(), |min2| min.min(min2)); + .map_or(min, |min2| min.min(min2)); self.forced_push_at(i, min, exit) })?; @@ -745,7 +690,7 @@ where T: From + Add + Ord, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + others.iter().map(|v| v.version()).sum(), + Version::ZERO + self.inner_version() + others.iter().map(|v| v.version()).sum(), )?; if others.is_empty() { @@ -765,7 +710,7 @@ where .iter_mut() .map(|iter| iter.unwrap_get_inner(i)) .max() - .map_or(max.clone(), |max2| max.max(max2)); + .map_or(max, |max2| max.max(max2)); self.forced_push_at(i, max, exit) })?; @@ -781,7 +726,7 @@ where ) -> Result<()> where T: Add + From + Div + From, - T2: StoredType, + T2: StoredRaw, f32: From + From, { self.compute_sma_(max_from, source, sma, exit, None) @@ -797,11 +742,11 @@ where ) -> Result<()> where T: Add + From + Div + From, - T2: StoredType, + T2: StoredRaw, f32: From + From, { self.validate_computed_version_or_reset_file( - Version::ONE + self.0.version() + source.version(), + Version::ONE + self.inner_version() + source.version(), )?; let index = max_from.min(I::from(self.len())); @@ -827,16 +772,16 @@ where let value = f32::from(value); let sma = T::from(if processed_values_count > sma { - let prev_sum = f32::from(prev.clone().unwrap()) * len as f32; + let prev_sum = f32::from(prev.unwrap()) * len as f32; let value_to_subtract = f32::from( other_iter.unwrap_get_inner_(i.unwrap_to_usize().checked_sub(sma).unwrap()), ); (prev_sum - value_to_subtract + value) / len as f32 } else { - (f32::from(prev.clone().unwrap()) * (len - 1) as f32 + value) / len as f32 + (f32::from(prev.unwrap()) * (len - 1) as f32 + value) / len as f32 }); - prev.replace(sma.clone()); + prev.replace(sma); self.forced_push_at(i, sma, exit) } else { self.forced_push_at(i, T::from(f32::NAN), exit) @@ -855,12 +800,12 @@ where ) -> Result<()> where I: CheckedSub, - T2: StoredType + Default, + T2: StoredRaw + Default, f32: From, T: From, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + source.version(), + Version::ZERO + self.inner_version() + source.version(), )?; let index = max_from.min(I::from(self.len())); @@ -891,7 +836,7 @@ where T: CheckedSub + Default, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + source.version(), + Version::ZERO + self.inner_version() + source.version(), )?; let index = max_from.min(I::from(self.len())); @@ -919,12 +864,12 @@ where ) -> Result<()> where I: CheckedSub, - T2: StoredType + Default, + T2: StoredRaw + Default, f32: From, T: From, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + source.version(), + Version::ZERO + self.inner_version() + source.version(), )?; let index = max_from.min(I::from(self.len())); @@ -955,12 +900,12 @@ where ) -> Result<()> where I: CheckedSub, - T2: StoredType + Default, + T2: StoredRaw + Default, f32: From, T: From, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + percentage_returns.version(), + Version::ZERO + self.inner_version() + percentage_returns.version(), )?; if days % 365 != 0 { @@ -983,16 +928,21 @@ where self.safe_flush(exit) } - pub fn compute_zscore( + pub fn compute_zscore( &mut self, max_from: I, - ratio: &impl AnyIterableVec, - sma: &impl AnyIterableVec, - sd: &impl AnyIterableVec, + ratio: &impl AnyIterableVec, + sma: &impl AnyIterableVec, + sd: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where - T: From, + T: From, + T2: StoredRaw + Sub + Div, + T3: StoredRaw, + T4: StoredRaw, + T2: StoredRaw, + f32: From + From + From, { let mut sma_iter = sma.iter(); let mut sd_iter = sd.iter(); @@ -1003,250 +953,17 @@ where |(i, ratio, ..)| { let sma = sma_iter.unwrap_get_inner(i); let sd = sd_iter.unwrap_get_inner(i); - (i, T::from((ratio - sma) / sd)) + (i, (ratio - sma) / sd) }, exit, ) } } -impl EagerVec { - pub fn compute_dca_stack_via_len( - &mut self, - max_from: DateIndex, - closes: &impl AnyIterableVec>, - len: usize, - exit: &Exit, - ) -> Result<()> { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + closes.version(), - )?; - - let mut other_iter = closes.iter(); - let mut prev = None; - - let index = max_from.min(DateIndex::from(self.len())); - closes.iter_at(index).try_for_each(|(i, closes)| { - let price = *closes.into_owned(); - let i_usize = i.unwrap_to_usize(); - if prev.is_none() { - if i_usize == 0 { - prev.replace(Sats::ZERO); - } else { - prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1)); - } - } - - let mut stack = Sats::ZERO; - - if price != Dollars::ZERO { - stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price)); - - if i_usize >= len { - let prev_price = *other_iter.unwrap_get_inner_(i_usize - len); - if prev_price != Dollars::ZERO { - stack = stack - .checked_sub(Sats::from(Bitcoin::from(DCA_AMOUNT / prev_price))) - .unwrap(); - } - } - } - - prev.replace(stack); - - self.forced_push_at(i, stack, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_dca_stack_via_from( - &mut self, - max_from: DateIndex, - closes: &impl AnyIterableVec>, - from: DateIndex, - exit: &Exit, - ) -> Result<()> { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + closes.version(), - )?; - - let mut prev = None; - - let index = max_from.min(DateIndex::from(self.len())); - closes.iter_at(index).try_for_each(|(i, closes)| { - let price = *closes.into_owned(); - let i_usize = i.unwrap_to_usize(); - if prev.is_none() { - if i_usize == 0 { - prev.replace(Sats::ZERO); - } else { - prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1)); - } - } - - let mut stack = Sats::ZERO; - - if price != Dollars::ZERO && i >= from { - stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price)); - } - - prev.replace(stack); - - self.forced_push_at(i, stack, exit) - })?; - - self.safe_flush(exit) - } -} - -impl EagerVec { - pub fn compute_dca_avg_price_via_len( - &mut self, - max_from: DateIndex, - stacks: &impl AnyIterableVec, - len: usize, - exit: &Exit, - ) -> Result<()> { - self.validate_computed_version_or_reset_file( - Version::ONE + self.0.version() + stacks.version(), - )?; - - let index = max_from.min(DateIndex::from(self.len())); - - let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap(); - - stacks.iter_at(index).try_for_each(|(i, stack)| { - let stack = stack.into_owned(); - let mut avg_price = Dollars::from(f64::NAN); - if i > first_price_date { - avg_price = DCA_AMOUNT - * len - .min(i.unwrap_to_usize() + 1) - .min(i.checked_sub(first_price_date).unwrap().unwrap_to_usize() + 1) - / Bitcoin::from(stack); - } - self.forced_push_at(i, avg_price, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_dca_avg_price_via_from( - &mut self, - max_from: DateIndex, - stacks: &impl AnyIterableVec, - from: DateIndex, - exit: &Exit, - ) -> Result<()> { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + stacks.version(), - )?; - - let index = max_from.min(DateIndex::from(self.len())); - - let from_usize = from.unwrap_to_usize(); - - stacks.iter_at(index).try_for_each(|(i, stack)| { - let stack = stack.into_owned(); - let mut avg_price = Dollars::from(f64::NAN); - if i >= from { - avg_price = - DCA_AMOUNT * (i.unwrap_to_usize() + 1 - from_usize) / Bitcoin::from(stack); - } - self.forced_push_at(i, avg_price, exit) - })?; - - self.safe_flush(exit) - } -} - -impl EagerVec -where - I: StoredIndex, -{ - pub fn compute_from_sats( - &mut self, - max_from: I, - sats: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + sats.version(), - )?; - - let index = max_from.min(I::from(self.len())); - sats.iter_at(index).try_for_each(|(i, sats)| { - let (i, v) = (i, Bitcoin::from(sats.into_owned())); - self.forced_push_at(i, v, exit) - })?; - - self.safe_flush(exit) - } -} - -impl EagerVec -where - I: StoredIndex, -{ - pub fn compute_from_bitcoin( - &mut self, - max_from: I, - bitcoin: &impl AnyIterableVec, - price: &impl AnyIterableVec>, - exit: &Exit, - ) -> Result<()> { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.0.version() + bitcoin.version(), - )?; - - let mut price_iter = price.iter(); - let index = max_from.min(I::from(self.len())); - bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| { - let dollars = price_iter.unwrap_get_inner(i); - let (i, v) = (i, *dollars * bitcoin.into_owned()); - self.forced_push_at(i, v, exit) - })?; - - self.safe_flush(exit) - } -} - -// impl EagerVec { -// pub fn compute_txindex_from_bitcoin( -// &mut self, -// max_from: TxIndex, -// bitcoin: &impl AnyIterableVec, -// i_to_height: &impl AnyIterableVec, -// price: &impl AnyIterableVec>, -// exit: &Exit, -// ) -> Result<()> { -// self.validate_computed_version_or_reset_file( -// Version::ZERO -// + self.0.version() -// + bitcoin.version() -// + i_to_height.version() -// + price.version(), -// )?; - -// let mut i_to_height_iter = i_to_height.iter(); -// let mut price_iter = price.iter(); -// let index = max_from.min(TxIndex::from(self.len())); -// bitcoin.iter_at(index).try_for_each(|(i, bitcoin, ..)| { -// let height = i_to_height_iter.unwrap_get_inner(i); -// let dollars = price_iter.unwrap_get_inner(height); -// let (i, v) = (i, *dollars * bitcoin.into_owned()); -// self.forced_push_at(i, v, exit) -// })?; - -// self.safe_flush(exit) -// } -// } - impl<'a, I, T> IntoIterator for &'a EagerVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { type Item = (I, Cow<'a, T>); type IntoIter = StoredVecIterator<'a, I, T>; @@ -1259,7 +976,7 @@ where impl AnyVec for EagerVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { #[inline] fn version(&self) -> Version { @@ -1287,15 +1004,93 @@ where } } +impl AnyStoredVec for EagerVec +where + I: StoredIndex, + T: StoredCompressed, +{ + fn file(&self) -> &File { + self.0.file() + } + + fn region_index(&self) -> usize { + self.0.region_index() + } + + fn header(&self) -> &Header { + self.0.header() + } + + fn mut_header(&mut self) -> &mut Header { + self.0.mut_header() + } + + fn flush(&mut self) -> Result<()> { + self.0.flush() + } + + fn stored_len(&self) -> usize { + self.0.stored_len() + } +} + +impl GenericStoredVec for EagerVec +where + I: StoredIndex, + T: StoredCompressed, +{ + #[inline] + fn read_(&self, index: usize, reader: &Reader) -> Result { + self.0.read_(index, reader) + } + + #[inline] + fn pushed(&self) -> &[T] { + self.0.pushed() + } + #[inline] + fn mut_pushed(&mut self) -> &mut Vec { + self.0.mut_pushed() + } + + #[inline] + fn holes(&self) -> &BTreeSet { + self.0.holes() + } + #[inline] + fn mut_holes(&mut self) -> &mut BTreeSet { + self.0.mut_holes() + } + + #[inline] + fn updated(&self) -> &BTreeMap { + self.0.updated() + } + #[inline] + fn mut_updated(&mut self) -> &mut BTreeMap { + self.0.mut_updated() + } + + #[inline] + fn truncate_if_needed(&mut self, index: I) -> Result<()> { + self.0.truncate_if_needed(index) + } + + #[inline] + fn reset(&mut self) -> Result<()> { + self.0.reset() + } +} + impl AnyIterableVec for EagerVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, { Box::new(self.0.into_iter()) } @@ -1304,7 +1099,7 @@ where impl AnyCollectableVec for EagerVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/lazy/lazy1.rs b/crates/brk_vecs/src/variants/lazy/lazy1.rs index 568192ca5..ab22a7461 100644 --- a/crates/brk_vecs/src/variants/lazy/lazy1.rs +++ b/crates/brk_vecs/src/variants/lazy/lazy1.rs @@ -1,10 +1,8 @@ use std::borrow::Cow; -use brk_core::{Result, Version}; - use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, - BoxedVecIterator, CollectableVec, StoredIndex, StoredType, + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, AnyBoxedIterableVec, + BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredRaw, Version, }; pub type ComputeFrom1 = @@ -17,21 +15,21 @@ where { name: String, version: Version, - source: BoxedAnyIterableVec, + source: AnyBoxedIterableVec, compute: ComputeFrom1, } impl LazyVecFrom1 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, { pub fn init( name: &str, version: Version, - source: BoxedAnyIterableVec, + source: AnyBoxedIterableVec, compute: ComputeFrom1, ) -> Self { if I::to_string() != S1I::to_string() { @@ -63,9 +61,9 @@ where impl<'a, I, T, S1I, S1T> Iterator for LazyVecFrom1Iterator<'a, I, T, S1I, S1T> where I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, { type Item = (I, Cow<'a, T>); @@ -85,9 +83,9 @@ where impl BaseVecIterator for LazyVecFrom1Iterator<'_, I, T, S1I, S1T> where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, { #[inline] fn mut_index(&mut self) -> &mut usize { @@ -108,9 +106,9 @@ where impl<'a, I, T, S1I, S1T> IntoIterator for &'a LazyVecFrom1 where I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, { type Item = (I, Cow<'a, T>); type IntoIter = LazyVecFrom1Iterator<'a, I, T, S1I, S1T>; @@ -127,9 +125,9 @@ where impl AnyVec for LazyVecFrom1 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, { fn version(&self) -> Version { self.version() @@ -156,9 +154,9 @@ where impl AnyIterableVec for LazyVecFrom1 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -171,9 +169,9 @@ where impl AnyCollectableVec for LazyVecFrom1 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/lazy/lazy2.rs b/crates/brk_vecs/src/variants/lazy/lazy2.rs index 0093f6454..d8ca303cd 100644 --- a/crates/brk_vecs/src/variants/lazy/lazy2.rs +++ b/crates/brk_vecs/src/variants/lazy/lazy2.rs @@ -1,10 +1,8 @@ use std::borrow::Cow; -use brk_core::{Result, Version}; - use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, - BoxedVecIterator, CollectableVec, StoredIndex, StoredType, + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, AnyBoxedIterableVec, + BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredRaw, Version, }; pub type ComputeFrom2 = for<'a> fn( @@ -21,25 +19,25 @@ where { name: String, version: Version, - source1: BoxedAnyIterableVec, - source2: BoxedAnyIterableVec, + source1: AnyBoxedIterableVec, + source2: AnyBoxedIterableVec, compute: ComputeFrom2, } impl LazyVecFrom2 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, { pub fn init( name: &str, version: Version, - source1: BoxedAnyIterableVec, - source2: BoxedAnyIterableVec, + source1: AnyBoxedIterableVec, + source2: AnyBoxedIterableVec, compute: ComputeFrom2, ) -> Self { if ([ @@ -82,11 +80,11 @@ where impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> where I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, { type Item = (I, Cow<'a, T>); @@ -105,11 +103,11 @@ impl BaseVecIterator for LazyVecFrom2Iterator<'_, I, T, S1I, S1T, S2I, S2T> where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, { #[inline] fn mut_index(&mut self) -> &mut usize { @@ -140,11 +138,11 @@ where impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVecFrom2 where I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, { type Item = (I, Cow<'a, T>); type IntoIter = LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>; @@ -162,11 +160,11 @@ where impl AnyVec for LazyVecFrom2 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, { fn version(&self) -> Version { self.version() @@ -203,11 +201,11 @@ where impl AnyIterableVec for LazyVecFrom2 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -220,11 +218,11 @@ where impl AnyCollectableVec for LazyVecFrom2 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/lazy/lazy3.rs b/crates/brk_vecs/src/variants/lazy/lazy3.rs index b94ad165b..08c326cce 100644 --- a/crates/brk_vecs/src/variants/lazy/lazy3.rs +++ b/crates/brk_vecs/src/variants/lazy/lazy3.rs @@ -1,10 +1,8 @@ use std::borrow::Cow; -use brk_core::{Result, Version}; - use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, - BoxedVecIterator, CollectableVec, StoredIndex, StoredType, + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, AnyBoxedIterableVec, + BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredRaw, Version, }; pub type ComputeFrom3 = for<'a> fn( @@ -23,29 +21,29 @@ where { name: String, version: Version, - source1: BoxedAnyIterableVec, - source2: BoxedAnyIterableVec, - source3: BoxedAnyIterableVec, + source1: AnyBoxedIterableVec, + source2: AnyBoxedIterableVec, + source3: AnyBoxedIterableVec, compute: ComputeFrom3, } impl LazyVecFrom3 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { pub fn init( name: &str, version: Version, - source1: BoxedAnyIterableVec, - source2: BoxedAnyIterableVec, - source3: BoxedAnyIterableVec, + source1: AnyBoxedIterableVec, + source2: AnyBoxedIterableVec, + source3: AnyBoxedIterableVec, compute: ComputeFrom3, ) -> Self { if ([ @@ -93,13 +91,13 @@ impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator for LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> where I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { type Item = (I, Cow<'a, T>); @@ -123,13 +121,13 @@ impl BaseVecIterator for LazyVecFrom3Iterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { #[inline] fn mut_index(&mut self) -> &mut usize { @@ -166,13 +164,13 @@ impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator for &'a LazyVecFrom3 where I: StoredIndex, - T: StoredType + 'a, + T: StoredRaw + 'a, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { type Item = (I, Cow<'a, T>); type IntoIter = LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; @@ -191,13 +189,13 @@ where impl AnyVec for LazyVecFrom3 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { fn version(&self) -> Version { self.version() @@ -240,13 +238,13 @@ impl AnyIterableVec for LazyVecFrom3 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -260,13 +258,13 @@ impl AnyCollectableVec for LazyVecFrom3 where I: StoredIndex, - T: StoredType, + T: StoredRaw, S1I: StoredIndex, - S1T: StoredType, + S1T: StoredRaw, S2I: StoredIndex, - S2T: StoredType, + S2T: StoredRaw, S3I: StoredIndex, - S3T: StoredType, + S3T: StoredRaw, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/mod.rs b/crates/brk_vecs/src/variants/mod.rs index 5e38d6fa6..db898658d 100644 --- a/crates/brk_vecs/src/variants/mod.rs +++ b/crates/brk_vecs/src/variants/mod.rs @@ -3,7 +3,6 @@ mod computed; mod eager; mod lazy; mod raw; -mod stamped; mod stored; pub use compressed::*; @@ -11,5 +10,4 @@ pub use computed::*; pub use eager::*; pub use lazy::*; pub use raw::*; -pub use stamped::*; pub use stored::*; diff --git a/crates/brk_vecs/src/variants/raw/header.rs b/crates/brk_vecs/src/variants/raw/header.rs index f8f1d9d31..61aa9844d 100644 --- a/crates/brk_vecs/src/variants/raw/header.rs +++ b/crates/brk_vecs/src/variants/raw/header.rs @@ -1,11 +1,10 @@ use std::sync::Arc; -use brk_core::{Error, Result, Version}; use parking_lot::RwLock; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{File, Stamp}; +use crate::{Error, File, Result, Stamp, Version}; use super::Format; @@ -168,7 +167,6 @@ impl HeaderInner { Immutable, KnownLayout, )] -#[repr(C)] pub struct ZeroCopyBool(u64); impl ZeroCopyBool { diff --git a/crates/brk_vecs/src/variants/raw/mod.rs b/crates/brk_vecs/src/variants/raw/mod.rs index 1ef9613b2..1c242e161 100644 --- a/crates/brk_vecs/src/variants/raw/mod.rs +++ b/crates/brk_vecs/src/variants/raw/mod.rs @@ -6,13 +6,13 @@ use std::{ sync::Arc, }; -use brk_core::{Error, Result, Version}; use parking_lot::RwLock; use rayon::prelude::*; +use zerocopy::FromBytes; use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec, - File, GenericStoredVec, Reader, StoredIndex, StoredType, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, BaseVecIterator, BoxedVecIterator, + CollectableVec, Error, File, GenericStoredVec, Reader, Result, StoredIndex, StoredRaw, Version, file::{Region, RegionReader}, }; @@ -44,7 +44,7 @@ pub struct RawVec { impl RawVec where I: StoredIndex, - T: StoredType, + T: StoredRaw, { /// Same as import but will reset the vec under certain errors, so be careful ! pub fn forced_import(file: &Arc, name: &str, mut version: Version) -> Result { @@ -91,9 +91,7 @@ where .create_reader(file) .read_all() .chunks(size_of::()) - .map(|b| -> Result { - Ok(usize::from_ne_bytes(brk_core::copy_first_8bytes(b)?)) - }) + .map(|b| -> Result { usize::read_from_bytes(b).map_err(|e| e.into()) }) .collect::>>()?, ) } else { @@ -139,22 +137,59 @@ where } } -impl GenericStoredVec for RawVec +impl Clone for RawVec { + fn clone(&self) -> Self { + Self { + file: self.file.clone(), + region: self.region.clone(), + region_index: self.region_index, + header: self.header.clone(), + name: self.name, + pushed: vec![], + updated: BTreeMap::new(), + has_stored_holes: false, + holes: BTreeSet::new(), + phantom: PhantomData, + } + } +} + +impl AnyVec for RawVec where I: StoredIndex, - T: StoredType, + T: StoredRaw, { #[inline] - fn read_(&self, index: usize, reader: &Reader) -> Result> { - let slice = reader.read( - (index * Self::SIZE_OF_T + HEADER_OFFSET) as u64, - (Self::SIZE_OF_T) as u64, - ); - T::try_read_from_bytes(slice) - .map(|v| Some(v)) - .map_err(Error::from) + fn version(&self) -> Version { + self.header.vec_version() } + #[inline] + fn name(&self) -> &str { + self.name + } + + #[inline] + fn len(&self) -> usize { + self.len_() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } +} + +impl AnyStoredVec for RawVec +where + I: StoredIndex, + T: StoredRaw, +{ fn header(&self) -> &Header { &self.header } @@ -174,33 +209,6 @@ where / Self::SIZE_OF_T } - #[inline] - fn pushed(&self) -> &[T] { - self.pushed.as_slice() - } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - &mut self.pushed - } - - #[inline] - fn holes(&self) -> &BTreeSet { - &self.holes - } - #[inline] - fn mut_holes(&mut self) -> &mut BTreeSet { - &mut self.holes - } - - #[inline] - fn updated(&self) -> &BTreeMap { - &self.updated - } - #[inline] - fn mut_updated(&mut self) -> &mut BTreeMap { - &mut self.updated - } - fn flush(&mut self) -> Result<()> { self.write_header_if_needed()?; @@ -271,6 +279,54 @@ where Ok(()) } + fn file(&self) -> &File { + &self.file + } + + fn region_index(&self) -> usize { + self.region_index + } +} + +impl GenericStoredVec for RawVec +where + I: StoredIndex, + T: StoredRaw, +{ + #[inline] + fn read_(&self, index: usize, reader: &Reader) -> Result { + T::read_from_prefix(reader.prefixed(index * Self::SIZE_OF_T + HEADER_OFFSET)) + .map(|(v, _)| v) + .map_err(Error::from) + } + + #[inline] + fn pushed(&self) -> &[T] { + self.pushed.as_slice() + } + #[inline] + fn mut_pushed(&mut self) -> &mut Vec { + &mut self.pushed + } + + #[inline] + fn holes(&self) -> &BTreeSet { + &self.holes + } + #[inline] + fn mut_holes(&mut self) -> &mut BTreeSet { + &mut self.holes + } + + #[inline] + fn updated(&self) -> &BTreeMap { + &self.updated + } + #[inline] + fn mut_updated(&mut self) -> &mut BTreeMap { + &mut self.updated + } + fn truncate_if_needed(&mut self, index: I) -> Result<()> { let index = index.to_usize()?; @@ -291,62 +347,6 @@ where fn reset(&mut self) -> Result<()> { self.reset_() } - - fn file(&self) -> &File { - &self.file - } - - fn region_index(&self) -> usize { - self.region_index - } -} - -impl AnyVec for RawVec -where - I: StoredIndex, - T: StoredType, -{ - #[inline] - fn version(&self) -> Version { - self.header.vec_version() - } - - #[inline] - fn name(&self) -> &str { - self.name - } - - #[inline] - fn len(&self) -> usize { - self.len_() - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl Clone for RawVec { - fn clone(&self) -> Self { - Self { - file: self.file.clone(), - region: self.region.clone(), - region_index: self.region_index, - header: self.header.clone(), - name: self.name, - pushed: vec![], - updated: BTreeMap::new(), - has_stored_holes: false, - holes: BTreeSet::new(), - phantom: PhantomData, - } - } } #[derive(Debug)] @@ -359,7 +359,7 @@ pub struct RawVecIterator<'a, I, T> { impl BaseVecIterator for RawVecIterator<'_, I, T> where I: StoredIndex, - T: StoredType, + T: StoredRaw, { #[inline] fn mut_index(&mut self) -> &mut usize { @@ -380,7 +380,7 @@ where impl<'a, I, T> Iterator for RawVecIterator<'a, I, T> where I: StoredIndex, - T: StoredType, + T: StoredRaw, { type Item = (I, Cow<'a, T>); @@ -404,7 +404,7 @@ where impl<'a, I, T> IntoIterator for &'a RawVec where I: StoredIndex, - T: StoredType, + T: StoredRaw, { type Item = (I, Cow<'a, T>); type IntoIter = RawVecIterator<'a, I, T>; @@ -421,7 +421,7 @@ where impl AnyIterableVec for RawVec where I: StoredIndex, - T: StoredType, + T: StoredRaw, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -434,7 +434,7 @@ where impl AnyCollectableVec for RawVec where I: StoredIndex, - T: StoredType, + T: StoredRaw, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/stamped/mod.rs b/crates/brk_vecs/src/variants/stamped/mod.rs index acb7663bf..d27faa392 100644 --- a/crates/brk_vecs/src/variants/stamped/mod.rs +++ b/crates/brk_vecs/src/variants/stamped/mod.rs @@ -1,10 +1,9 @@ use std::{borrow::Cow, cmp::Ordering, fmt::Debug, sync::Arc}; -use brk_core::{Error, Result, Version}; - use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, File, Format, - GenericStoredVec, Header, StoredIndex, StoredType, StoredVec, file::Reader, + AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Error, File, + Format, GenericStoredVec, Header, Result, StoredCompressed, StoredIndex, StoredVec, Version, + file::Reader, }; use super::StoredVecIterator; @@ -19,7 +18,7 @@ pub struct StampedVec(StoredVec); impl StampedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { pub fn forced_import( file: &Arc, @@ -36,7 +35,10 @@ where } #[inline] - pub fn get_or_read(&self, index: I, reader: &Reader) -> Result>> { + pub fn get_or_read<'a, 'b>(&'a self, index: I, reader: &'b Reader) -> Result>> + where + 'a: 'b, + { self.0.get_or_read(index, reader) } @@ -142,7 +144,7 @@ where impl AnyVec for StampedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { #[inline] fn version(&self) -> Version { @@ -178,7 +180,7 @@ pub trait AnyStampedVec: AnyVec { impl AnyStampedVec for StampedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn stamp(&self) -> Stamp { self.0.header().stamp() @@ -192,7 +194,7 @@ where impl<'a, I, T> IntoIterator for &'a StampedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { type Item = (I, Cow<'a, T>); type IntoIter = StoredVecIterator<'a, I, T>; @@ -205,7 +207,7 @@ where impl AnyIterableVec for StampedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -218,7 +220,7 @@ where impl AnyCollectableVec for StampedVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn collect_range_serde_json( &self, diff --git a/crates/brk_vecs/src/variants/stored/format.rs b/crates/brk_vecs/src/variants/stored/format.rs index ab5e5452c..a458d2dae 100644 --- a/crates/brk_vecs/src/variants/stored/format.rs +++ b/crates/brk_vecs/src/variants/stored/format.rs @@ -1,12 +1,10 @@ use std::{fs, io, path::Path}; -use brk_core::{Error, Result}; -use clap_derive::ValueEnum; use serde::{Deserialize, Serialize}; -#[derive( - Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum, -)] +use crate::{Error, Result}; + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub enum Format { Compressed, #[default] diff --git a/crates/brk_vecs/src/variants/stored/mod.rs b/crates/brk_vecs/src/variants/stored/mod.rs index 4708f0ad5..d1064188f 100644 --- a/crates/brk_vecs/src/variants/stored/mod.rs +++ b/crates/brk_vecs/src/variants/stored/mod.rs @@ -4,11 +4,10 @@ use std::{ sync::Arc, }; -use brk_core::{Result, Version}; - use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec, - File, GenericStoredVec, Header, StoredIndex, StoredType, file::Reader, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, BaseVecIterator, BoxedVecIterator, + CollectableVec, File, GenericStoredVec, Header, Result, StoredCompressed, StoredIndex, Version, + file::Reader, }; use super::{CompressedVec, CompressedVecIterator, RawVec, RawVecIterator}; @@ -26,7 +25,7 @@ pub enum StoredVec { impl StoredVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { pub fn forced_import( file: &Arc, @@ -50,10 +49,46 @@ where } } -impl GenericStoredVec for StoredVec +impl AnyVec for StoredVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, +{ + #[inline] + fn version(&self) -> Version { + match self { + StoredVec::Raw(v) => v.version(), + StoredVec::Compressed(v) => v.version(), + } + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + #[inline] + fn len(&self) -> usize { + self.pushed_len() + self.stored_len() + } + + fn name(&self) -> &str { + match self { + StoredVec::Raw(v) => v.name(), + StoredVec::Compressed(v) => v.name(), + } + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } +} + +impl AnyStoredVec for StoredVec +where + I: StoredIndex, + T: StoredCompressed, { #[inline] fn file(&self) -> &File { @@ -71,14 +106,6 @@ where } } - #[inline] - fn read_(&self, index: usize, reader: &Reader) -> Result> { - match self { - StoredVec::Raw(v) => v.read_(index, reader), - StoredVec::Compressed(v) => v.read_(index, reader), - } - } - #[inline] fn header(&self) -> &Header { match self { @@ -103,6 +130,27 @@ where } } + fn flush(&mut self) -> Result<()> { + match self { + StoredVec::Raw(v) => v.flush(), + StoredVec::Compressed(v) => v.flush(), + } + } +} + +impl GenericStoredVec for StoredVec +where + I: StoredIndex, + T: StoredCompressed, +{ + #[inline] + fn read_(&self, index: usize, reader: &Reader) -> Result { + match self { + StoredVec::Raw(v) => v.read_(index, reader), + StoredVec::Compressed(v) => v.read_(index, reader), + } + } + #[inline] fn pushed(&self) -> &[T] { match self { @@ -148,13 +196,7 @@ where } } - fn flush(&mut self) -> Result<()> { - match self { - StoredVec::Raw(v) => v.flush(), - StoredVec::Compressed(v) => v.flush(), - } - } - + #[inline] fn truncate_if_needed(&mut self, index: I) -> Result<()> { match self { StoredVec::Raw(v) => v.truncate_if_needed(index), @@ -162,6 +204,7 @@ where } } + #[inline] fn reset(&mut self) -> Result<()> { match self { StoredVec::Raw(v) => v.reset(), @@ -170,42 +213,6 @@ where } } -impl AnyVec for StoredVec -where - I: StoredIndex, - T: StoredType, -{ - #[inline] - fn version(&self) -> Version { - match self { - StoredVec::Raw(v) => v.version(), - StoredVec::Compressed(v) => v.version(), - } - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn len(&self) -> usize { - self.pushed_len() + self.stored_len() - } - - fn name(&self) -> &str { - match self { - StoredVec::Raw(v) => v.name(), - StoredVec::Compressed(v) => v.name(), - } - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - #[derive(Debug)] pub enum StoredVecIterator<'a, I, T> { Raw(RawVecIterator<'a, I, T>), @@ -215,7 +222,7 @@ pub enum StoredVecIterator<'a, I, T> { impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T> where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { @@ -229,7 +236,7 @@ where impl BaseVecIterator for StoredVecIterator<'_, I, T> where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { #[inline] fn mut_index(&mut self) -> &mut usize { @@ -258,7 +265,7 @@ where impl<'a, I, T> IntoIterator for &'a StoredVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { type Item = (I, Cow<'a, T>); type IntoIter = StoredVecIterator<'a, I, T>; @@ -274,7 +281,7 @@ where impl AnyIterableVec for StoredVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where @@ -287,7 +294,7 @@ where impl AnyCollectableVec for StoredVec where I: StoredIndex, - T: StoredType, + T: StoredCompressed, { fn collect_range_serde_json( &self, diff --git a/crates/brk_core/src/structs/version.rs b/crates/brk_vecs/src/version.rs similarity index 95% rename from crates/brk_core/src/structs/version.rs rename to crates/brk_vecs/src/version.rs index e05109b98..eff79c0c6 100644 --- a/crates/brk_core/src/structs/version.rs +++ b/crates/brk_vecs/src/version.rs @@ -44,8 +44,11 @@ impl Version { Self(self.0.swap_bytes()) } - /// Ok(true) if existed and is same - /// Ok(false) if didn't exist + /// + /// Ok(true) if existed and is same. + /// + /// Ok(false) if didn't exist. + /// pub fn validate(&self, path: &Path) -> Result { if let Ok(prev_version) = Version::try_from(path) { if prev_version != *self { diff --git a/crates/brk_vecs_macros/Cargo.toml b/crates/brk_vecs_macros/Cargo.toml new file mode 100644 index 000000000..e4254bd58 --- /dev/null +++ b/crates/brk_vecs_macros/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "brk_vecs_macros" +description = "Macros for brk_vecs" +keywords = ["vec", "disk", "data"] +categories = ["database"] +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true + +[lib] +proc-macro = true + +[dependencies] +syn = "2.0" +quote = "1.0" diff --git a/crates/brk_vecs_macros/src/lib.rs b/crates/brk_vecs_macros/src/lib.rs new file mode 100644 index 000000000..081321103 --- /dev/null +++ b/crates/brk_vecs_macros/src/lib.rs @@ -0,0 +1,57 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{Data, DataStruct, DeriveInput, Fields, parse_macro_input}; + +#[proc_macro_derive(StoredCompressed)] +pub fn derive_stored_compressed(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let struct_name = &input.ident; + let generics = &input.generics; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let inner_type = match &input.data { + Data::Struct(DataStruct { + fields: Fields::Unnamed(fields), + .. + }) if fields.unnamed.len() == 1 => &fields.unnamed[0].ty, + _ => { + return syn::Error::new_spanned( + &input.ident, + "StoredCompressed can only be derived for single-field tuple structs", + ) + .to_compile_error() + .into(); + } + }; + + // Check if we have generic parameters + let has_generics = !generics.params.is_empty(); + + let expanded = if has_generics { + // Generic case: add where clause constraint + let where_clause = if where_clause.is_some() { + quote! { #where_clause #inner_type: StoredCompressed, } + } else { + quote! { where #inner_type: StoredCompressed, } + }; + + quote! { + impl #impl_generics ::brk_vecs::TransparentStoredCompressed<<#inner_type as StoredCompressed>::NumberType> for #struct_name #ty_generics #where_clause {} + + impl #impl_generics StoredCompressed for #struct_name #ty_generics #where_clause { + type NumberType = <#inner_type as StoredCompressed>::NumberType; + } + } + } else { + // Non-generic case: use the original approach + quote! { + impl ::brk_vecs::TransparentStoredCompressed<<#inner_type as StoredCompressed>::NumberType> for #struct_name {} + + impl StoredCompressed for #struct_name { + type NumberType = <#inner_type as StoredCompressed>::NumberType; + } + } + }; + + TokenStream::from(expanded) +} diff --git a/websites/default/scripts/options.js b/websites/default/scripts/options.js index 59ee685cf..2236c50bb 100644 --- a/websites/default/scripts/options.js +++ b/websites/default/scripts/options.js @@ -3639,14 +3639,6 @@ function createPartialOptions({ env, colors, vecIdToIndexes }) { name: "Discord", url: () => "https://discord.com/invite/HaR3wpH3nr", }, - { - name: "Bluesky", - url: () => "https://bsky.app/profile/bitcoinresearchkit.org", - }, - { - name: "x", - url: () => "https://x.com/brkdotorg", - }, ], }, {