diff --git a/Cargo.lock b/Cargo.lock index 68c48520e..ca61ac343 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,6 +156,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" @@ -164,9 +170,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-compression" -version = "0.4.25" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4" +checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" dependencies = [ "brotli", "flate2", @@ -339,6 +345,21 @@ dependencies = [ "virtue", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitcoin" version = "0.32.6" @@ -432,6 +453,19 @@ dependencies = [ "serde", ] +[[package]] +name = "blake3" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -443,7 +477,7 @@ dependencies = [ [[package]] name = "brk" -version = "0.0.71" +version = "0.0.80" dependencies = [ "brk_bundler", "brk_cli", @@ -457,14 +491,13 @@ dependencies = [ "brk_mcp", "brk_parser", "brk_server", - "brk_state", "brk_store", "brk_vec", ] [[package]] name = "brk_bundler" -version = "0.0.71" +version = "0.0.80" dependencies = [ "brk_rolldown", "log", @@ -475,7 +508,7 @@ dependencies = [ [[package]] name = "brk_cli" -version = "0.0.71" +version = "0.0.80" dependencies = [ "bitcoincore-rpc", "brk_computer", @@ -498,8 +531,9 @@ dependencies = [ [[package]] name = "brk_computer" -version = "0.0.71" +version = "0.0.80" dependencies = [ + "bincode", "bitcoin", "bitcoincore-rpc", "brk_core", @@ -508,18 +542,22 @@ dependencies = [ "brk_indexer", "brk_logger", "brk_parser", - "brk_state", + "brk_store", "brk_vec", "color-eyre", + "derive_deref", + "either", "fjall", - "jiff", "log", "rayon", + "serde", + "zerocopy", + "zerocopy-derive", ] [[package]] name = "brk_core" -version = "0.0.71" +version = "0.0.80" dependencies = [ "bincode", "bitcoin", @@ -540,7 +578,7 @@ dependencies = [ [[package]] name = "brk_exit" -version = "0.0.71" +version = "0.0.80" dependencies = [ "brk_logger", "ctrlc", @@ -549,7 +587,7 @@ dependencies = [ [[package]] name = "brk_fetcher" -version = "0.0.71" +version = "0.0.80" dependencies = [ "brk_core", "brk_logger", @@ -561,7 +599,7 @@ dependencies = [ [[package]] name = "brk_indexer" -version = "0.0.71" +version = "0.0.80" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -579,7 +617,7 @@ dependencies = [ [[package]] name = "brk_interface" -version = "0.0.71" +version = "0.0.80" dependencies = [ "brk_computer", "brk_core", @@ -588,7 +626,7 @@ dependencies = [ "brk_vec", "color-eyre", "derive_deref", - "schemars 1.0.1", + "schemars 1.0.4", "serde", "serde_json", "serde_with", @@ -597,7 +635,7 @@ dependencies = [ [[package]] name = "brk_logger" -version = "0.0.71" +version = "0.0.80" dependencies = [ "color-eyre", "env_logger", @@ -607,18 +645,17 @@ dependencies = [ [[package]] name = "brk_mcp" -version = "0.0.71" +version = "0.0.80" dependencies = [ "axum", "brk_interface", "brk_rmcp", "log", - "tracing", ] [[package]] name = "brk_parser" -version = "0.0.71" +version = "0.0.80" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -633,9 +670,9 @@ dependencies = [ [[package]] name = "brk_rmcp" -version = "0.1.8" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26db177d2933b727fe5f53b52d5ebead13d57f94ed8637717117a8bb1a3f1bae" +checksum = "0dde96460b07f97f8522cef62fa5e9d5271a5183b13e6d75379ab43c1fb0b106" dependencies = [ "base64 0.22.1", "brk_rmcp-macros", @@ -648,7 +685,7 @@ dependencies = [ "paste", "pin-project-lite", "rand 0.9.1", - "schemars 1.0.1", + "schemars 1.0.4", "serde", "serde_json", "sse-stream", @@ -663,9 +700,9 @@ dependencies = [ [[package]] name = "brk_rmcp-macros" -version = "0.1.8" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4557e65ffea1466260a6ab13e3ba54bcee69c39576e4bfb8b81e50b375924540" +checksum = "48a3cb1aadff2f3aaaed76d17e71d6b828cafbd3fe9fbaba07d31b8ed10e63c2" dependencies = [ "darling", "proc-macro2", @@ -676,9 +713,9 @@ dependencies = [ [[package]] name = "brk_rolldown" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a28608031195a1df827c2e9ac11f7b26acc2e447e0999e9914520941dd93a3" +checksum = "e84c9b257b6d6e15e6598f320fbe6dcc8615827dbd207aa73378a86723485e9f" dependencies = [ "anyhow", "append-only-vec", @@ -689,9 +726,9 @@ dependencies = [ "brk_rolldown_ecmascript_utils", "brk_rolldown_error", "brk_rolldown_fs", - "brk_rolldown_loader_utils", "brk_rolldown_plugin", "brk_rolldown_plugin_data_uri", + "brk_rolldown_plugin_hmr", "brk_rolldown_resolver", "brk_rolldown_rstr", "brk_rolldown_sourcemap", @@ -703,12 +740,13 @@ dependencies = [ "css-module-lexer", "dunce", "futures", - "indexmap 2.9.0", + "indexmap 2.10.0", "itertools", "itoa", "memchr", "notify", "oxc", + "oxc_ecmascript", "oxc_index", "petgraph", "rustc-hash", @@ -722,9 +760,9 @@ dependencies = [ [[package]] name = "brk_rolldown_common" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bf1ae1dbbde9bd49b1202936e5ae4ecad753d0ddfbbd0c4b06a9c48d3889276" +checksum = "ca6da445ded7b4b07e9b48fd1855091ca87ee83777ae49326644019c0255278b" dependencies = [ "anyhow", "arcstr", @@ -741,7 +779,9 @@ dependencies = [ "derive_more", "fast-glob", "itertools", + "num-bigint", "oxc", + "oxc_ecmascript", "oxc_index", "oxc_resolver", "rustc-hash", @@ -753,10 +793,11 @@ dependencies = [ [[package]] name = "brk_rolldown_debug" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1966b2aa72cbb9442a3158b818d6352d344270e3d148d9e0c4eb07b355cef959" +checksum = "a554a589c6027241dc5735ed2f9387ca4e6771b5a3c68b974b086cde857475a9" dependencies = [ + "blake3", "brk_rolldown_debug_action", "dashmap", "rustc-hash", @@ -768,9 +809,9 @@ dependencies = [ [[package]] name = "brk_rolldown_debug_action" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b094d3b0275f415a6386251b9485c941edbf38833ae03db3da32aae803593de" +checksum = "4f0f12e60c7f66ff8efe64dfb47da84ce6216e9c969442f70fb0edf4f3d7d5d1" dependencies = [ "serde", "ts-rs", @@ -778,9 +819,9 @@ dependencies = [ [[package]] name = "brk_rolldown_ecmascript" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11ff89014347e7090ebf569c6ac7b32482690d5679c5073fb9050a7778d890b" +checksum = "d0e6d41996222d97ada7efb7f00be28becae01fd6dd68ced9b2d659f9811f3ff" dependencies = [ "arcstr", "brk_rolldown_error", @@ -791,9 +832,9 @@ dependencies = [ [[package]] name = "brk_rolldown_ecmascript_utils" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938a6078497d6841166f63354e9bc9e029c2afb80a5344ee8c416359e80e66bc" +checksum = "43216eb0cd76628bfc878a34c26f8deaba7f1ab2a3afed3912670c53eae67c25" dependencies = [ "brk_rolldown_common", "oxc", @@ -802,9 +843,9 @@ dependencies = [ [[package]] name = "brk_rolldown_error" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71f9aa1cd9aacf8d66e2523a76dc8302718c344b11d3f81bec61e2a4fa97361" +checksum = "744db025ff88c465df4d4faed5e3c8f3ef91ae9b3183bbc80c62d41dc636c4f5" dependencies = [ "anyhow", "arcstr", @@ -822,30 +863,19 @@ dependencies = [ [[package]] name = "brk_rolldown_fs" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c752f5bdf13702e5a515af3699b72206c4b4a1b87844eb5dc28e01ca065912dd" +checksum = "9c38de0fd6c358c873e4a70e0054b65b6e08262f0676c47394c4f9f392f11c42" dependencies = [ "oxc_resolver", "vfs", ] -[[package]] -name = "brk_rolldown_loader_utils" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8397251ae82b01b89c723d30a67988e307dc76802e854f4a7c4824fd6560c1b" -dependencies = [ - "anyhow", - "brk_rolldown_common", - "serde_json", -] - [[package]] name = "brk_rolldown_plugin" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5be2dc55688443ebe37ef38d4e2912f6ec9a31c9490d3cef91f5cedf511f49f" +checksum = "66b3739b2e98ee755130b5d624135d2a8623051341a1b942ac1aed4fe65d5a95" dependencies = [ "anyhow", "arcstr", @@ -873,9 +903,9 @@ dependencies = [ [[package]] name = "brk_rolldown_plugin_data_uri" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc4e963da4c646a8d9396042a1dd3414850bbb8411df9f052de6e69cd6d9495" +checksum = "36f094093ec33d62c17012772db763bd1f3b9d8ee5bcbc374e6f8e9d66555279" dependencies = [ "arcstr", "base64-simd", @@ -886,10 +916,22 @@ dependencies = [ ] [[package]] -name = "brk_rolldown_resolver" -version = "0.0.1" +name = "brk_rolldown_plugin_hmr" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1955303e224ad9284a40415dce7ab10de6c8675d51f6b173ec664fda069b50b4" +checksum = "37ed61d459d0aa300b568ac2cf2153c7c4e5c510f901b9efce744f0d5134cc7f" +dependencies = [ + "arcstr", + "brk_rolldown_common", + "brk_rolldown_plugin", + "oxc", +] + +[[package]] +name = "brk_rolldown_resolver" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ae763e1c514bcfec0dd87be351f5000658feb0e4b1bcb5f199c60e6cbe7150" dependencies = [ "arcstr", "brk_rolldown_common", @@ -903,18 +945,18 @@ dependencies = [ [[package]] name = "brk_rolldown_rstr" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88fb9fe9f824838c4b74ecd63d9d4109ff24fafb43056c191fed58a778167230" +checksum = "a251b8c38c8b6c177319a5d1f1fb609a120f35e73a52af575792f94dfd42fd1a" dependencies = [ "oxc", ] [[package]] name = "brk_rolldown_sourcemap" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "607e6e8442cb0796834581e6922c5e1c648b6f139b135b81b31bed920c14ae10" +checksum = "a5465d4ebe0e0801f163d19779bf9858051aff2c291efc64fa6dde83d2e3238d" dependencies = [ "brk_rolldown_utils", "memchr", @@ -925,18 +967,18 @@ dependencies = [ [[package]] name = "brk_rolldown_std_utils" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4606e3131c5f4f7c9a0b02c3cd12d8b74501359d71b1e9af53bf6542934e02c7" +checksum = "50a28045a4b7ec4c5f8ff06a170d21e0c9a1d41e2baecc8348241f4d3d69a165" dependencies = [ "regex", ] [[package]] name = "brk_rolldown_tracing" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602c2dd4104cfc60f9207ea3f070ad1ed1f71f5676118ec4f0dd4be93f2eced9" +checksum = "35845acebabca843c6b2c3e1928c393c588ff0ba4bb4bcfcd571c1cc11b13d89" dependencies = [ "tracing", "tracing-chrome", @@ -945,9 +987,9 @@ dependencies = [ [[package]] name = "brk_rolldown_utils" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c213fcc1635c4c9de882a10beb295743bd6baa551f789bde0aa16bab5c3919a" +checksum = "22e3b2a75f83b012138bc63c16e238dfebd9484f99fb77a35fe96a10ff6ef437" dependencies = [ "anyhow", "arcstr", @@ -960,7 +1002,7 @@ dependencies = [ "fast-glob", "form_urlencoded", "futures", - "indexmap 2.9.0", + "indexmap 2.10.0", "infer", "itoa", "memchr", @@ -976,12 +1018,13 @@ dependencies = [ "serde_json", "simdutf8", "sugar_path", + "tokio", "xxhash-rust", ] [[package]] name = "brk_server" -version = "0.0.71" +version = "0.0.80" dependencies = [ "axum", "bitcoincore-rpc", @@ -1010,32 +1053,21 @@ dependencies = [ "zip", ] -[[package]] -name = "brk_state" -version = "0.0.71" -dependencies = [ - "bincode", - "brk_core", - "derive_deref", - "serde", - "zerocopy", - "zerocopy-derive", -] - [[package]] name = "brk_store" -version = "0.0.71" +version = "0.0.80" dependencies = [ "brk_core", "byteview", "fjall", + "log", ] [[package]] name = "brk_string_wizard" -version = "0.0.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455a8c39f673a5eedbeee3d131f87abb6a8a405476b9f0766cfdd7d047750fc0" +checksum = "a3e6948e03baceed9fc3e8b3274bf9fefdf56bcdd517a6ff4ecf1843a59178f7" dependencies = [ "oxc_index", "oxc_sourcemap", @@ -1045,7 +1077,7 @@ dependencies = [ [[package]] name = "brk_vec" -version = "0.0.71" +version = "0.0.80" dependencies = [ "arc-swap", "brk_core", @@ -1085,9 +1117,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" dependencies = [ "allocator-api2", ] @@ -1118,37 +1150,27 @@ checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" [[package]] name = "bzip2" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", + "libbz2-rs-sys", ] [[package]] name = "castaway" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" dependencies = [ "rustversion", ] [[package]] name = "cc" -version = "1.2.27" +version = "1.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" dependencies = [ "jobserver", "libc", @@ -1205,18 +1227,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" dependencies = [ "anstream", "anstyle", @@ -1226,9 +1248,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck", "proc-macro2", @@ -1242,6 +1264,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "clean-path" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaa6b4b263a5d737e9bf6b7c09b72c41a5480aec4d7219af827f6564e950b6a5" + [[package]] name = "color-eyre" version = "0.6.5" @@ -1305,6 +1333,15 @@ version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" +[[package]] +name = "concurrent_lru" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feb5cb312f774e8a24540e27206db4e890f7d488563671d24a16389cf4c2e4e" +dependencies = [ + "once_cell", +] + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -1334,9 +1371,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] @@ -1579,6 +1616,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enum_dispatch" version = "0.3.13" @@ -1640,6 +1683,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + [[package]] name = "fast-glob" version = "0.4.5" @@ -1848,11 +1902,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", - "wasm-bindgen", ] [[package]] @@ -1993,9 +2045,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" +checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df" dependencies = [ "bytes", "futures-core", @@ -2056,9 +2108,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", "hashbrown 0.15.4", @@ -2112,6 +2164,17 @@ dependencies = [ "compare", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2241,6 +2304,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "libbz2-rs-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb" + [[package]] name = "libc" version = "0.2.174" @@ -2269,9 +2338,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638" dependencies = [ "bitflags 2.9.1", "libc", @@ -2362,9 +2431,9 @@ checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memmap2" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28" dependencies = [ "libc", ] @@ -2386,9 +2455,9 @@ dependencies = [ [[package]] name = "minreq" -version = "2.13.4" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d2aaba477837b46ec1289588180fabfccf0c3b1d1a0c6b1866240cd6cd5ce9" +checksum = "84885312a86831bff4a3cb04a1e54a3f698407e3274c83249313f194d3e0b678" dependencies = [ "log", "rustls", @@ -2410,6 +2479,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.30.1" @@ -2439,12 +2517,11 @@ checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" [[package]] name = "notify" -version = "8.0.0" +version = "8.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" +checksum = "3163f59cd3fa0e9ef8c32f242966a7b9994fd7378366099593e0e73077cd8c97" dependencies = [ "bitflags 2.9.1", - "filetime", "fsevent-sys", "inotify", "kqueue", @@ -2453,7 +2530,7 @@ dependencies = [ "mio", "notify-types", "walkdir", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2547,9 +2624,9 @@ checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" [[package]] name = "oxc" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e31318da7866fb30c000877ae98c30d1f76212e4bdc6c088c5cb847e5d40954" +checksum = "e3692c2bbd698b3b2b18c4062b9df584f390d6154bb3815147d77b61354e3f0b" dependencies = [ "oxc_allocator", "oxc_ast", @@ -2571,10 +2648,11 @@ dependencies = [ [[package]] name = "oxc-browserslist" -version = "2.0.6" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7bcc31e3014fa4226346b8596cda0f5cb5797045fa2760c6b871407db5642c" +checksum = "20605046d78dbba8ed6c1c10b5ee21a83d265825943bce8bc40d17b4c48d5aca" dependencies = [ + "bincode", "nom", "rustc-hash", "serde", @@ -2585,9 +2663,9 @@ dependencies = [ [[package]] name = "oxc-miette" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b2c44324a4372caf6e3128a22744263c973e809fc598db3749ef3ff5e9fed4" +checksum = "00cb1a49ec377f62606cbf047794efd37d668dbcbcefaeb5bf43f89b3c391418" dependencies = [ "cfg-if", "owo-colors", @@ -2599,9 +2677,9 @@ dependencies = [ [[package]] name = "oxc-miette-derive" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd3da01a295024fa79e3b4aba14b590d91256a274ff29cc5ee8f55183b2df24" +checksum = "1739910e9871fe8d6e311f80fb2793756335aec97b0f985e778cbf4bc5cf574f" dependencies = [ "proc-macro2", "quote", @@ -2610,9 +2688,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4cb225affc487a1bc867455220d5427d0f2a35ed25d896f99bb3b912d49fb9e" +checksum = "905a588c0a12c71ddb94d4eaee6ea3770f9132017956936777763ebcde66d58c" dependencies = [ "allocator-api2", "bumpalo", @@ -2625,9 +2703,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ced8dcc14d588fa32594d70ff8f194712036d02d7a96718bce38abbfec72ed6" +checksum = "d203d52b5de3eba556625cac42290ac81f9995c82009b9f6ec041237f068ba9b" dependencies = [ "bitflags 2.9.1", "oxc_allocator", @@ -2641,9 +2719,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43494643bd6d76a62446c58ae98568bf630c0bdd90726d7956d3f8e1e17f5906" +checksum = "b31a7f8a301f434bd73f7735f02d7d823c2d4a83bd1ce1b04bfd3b0b50c5a054" dependencies = [ "phf", "proc-macro2", @@ -2653,9 +2731,9 @@ dependencies = [ [[package]] name = "oxc_ast_visit" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761d7f84b87080cf337c843fa60a1c3f50eb53eba0ec1e42d8758e99cd834031" +checksum = "c353d88dd55b1beb8338a99d908f2d5828f336b82821298d090ff4247c7a9b15" dependencies = [ "oxc_allocator", "oxc_ast", @@ -2666,9 +2744,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d561bbe6e05cd3be34802a22e799c3c9bd5987daee2dbb51b2561ab78d39420" +checksum = "4e1a01365605258dfafd0cb85b4340f20676eb726baa6feb799ea2fbee66739b" dependencies = [ "bitflags 2.9.1", "itertools", @@ -2681,9 +2759,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d694cadd4d0d86831197d720ad245ea3c82366db778230dfa8d80475d2049c80" +checksum = "bcb7de0fc40c9a935e8483692f7d17371d0d9f001ef92d0734094096851ef86a" dependencies = [ "bitflags 2.9.1", "cow-utils", @@ -2702,9 +2780,9 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebcebde288c0dbc9b8bb7ecf8d4eb3d64c6f122609fbca9f89dce356786fa19" +checksum = "7087f6bf6910ef41df29d07eb67b163207aafd7efa0801d3b57c2e072c8b9ed1" dependencies = [ "ropey", "rustversion", @@ -2712,19 +2790,20 @@ dependencies = [ [[package]] name = "oxc_diagnostics" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136bd60c8d25e6c1b879a01d5f52ad260385afa2d74b0aa81c1cbc9680b468ef" +checksum = "4fe0cbd689125f1510f44235fee96887f1f1f42a3420afc802db60ff654030cf" dependencies = [ "cow-utils", "oxc-miette", + "percent-encoding", ] [[package]] name = "oxc_ecmascript" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "405573ecc303a3e9852b2873aad30e8249b3e3cb668cf265f53a9072dada8d50" +checksum = "2141d6765bc6e75ff286aeb6f580f1ceeea40c48d23d07b943e1fb92367a8968" dependencies = [ "num-bigint", "num-traits", @@ -2735,9 +2814,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd0073f1b212fda3c1e37728db069bf22d136a90f58da4f6214a6ada7552dde" +checksum = "1216c5ef637573a66b9e4006a6edb432274799bd9e55059607a52aaa6bdbad9a" dependencies = [ "itoa", "oxc_data_structures", @@ -2756,9 +2835,9 @@ dependencies = [ [[package]] name = "oxc_isolated_declarations" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc08c216028563ebe9f35e9ff661c24d5f885735775d21fc00d579f7a06db10a" +checksum = "bba6deb4abf0dab2818cb73254535656607ed574f0bd271331967269a714ebb1" dependencies = [ "bitflags 2.9.1", "oxc_allocator", @@ -2773,9 +2852,9 @@ dependencies = [ [[package]] name = "oxc_mangler" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b301242a458a38572ec3ffd73d836de068019d605a49360f0c80ff9fcdbb0485" +checksum = "30668818436a7791798112326f43fcfd94a65a23364e9cb24939f77d7dc7ded2" dependencies = [ "fixedbitset", "itertools", @@ -2790,9 +2869,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d130925e189994a4695d7379f14fd427f347642741d8180baa0b8a6699607b" +checksum = "f2d92fc42e0f6ed63d689f0070c398e53fe04d5a553c681a3085798dfacb2ef4" dependencies = [ "cow-utils", "oxc_allocator", @@ -2812,9 +2891,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0b65b7f87759287bc7c8d394733cda9bca5e14fe7b71388932c926f8cde67b" +checksum = "55a30d3f398929eaac185d328e43233f857626dedd8be5b8c78bfc0c63a32d7c" dependencies = [ "bitflags 2.9.1", "cow-utils", @@ -2835,9 +2914,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0899e918e7da993922ac9d85a7cacefc5519afbab002a4e239aa0a8dc2201297" +checksum = "7e40e3de8fc6bb9091ec5d364a951405bca49ada74c5b0250e0cbb33583c9efc" dependencies = [ "bitflags 2.9.1", "oxc_allocator", @@ -2851,15 +2930,16 @@ dependencies = [ [[package]] name = "oxc_resolver" -version = "11.2.1" +version = "11.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9fbd425311540ada87997dbec6e9a2e0bab00b7e82db8e6ef79a0d0a9cfce1" +checksum = "e300c931167c1f902f8029a7aac00372cc1742c6a0c9f625a8b67dd337f557c7" dependencies = [ "cfg-if", - "indexmap 2.9.0", + "indexmap 2.10.0", "json-strip-comments", "once_cell", "papaya", + "pnp", "rustc-hash", "serde", "serde_json", @@ -2870,9 +2950,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b916b0e8bf2045895726a5069947789226eaae47e0f27412956163e02bd20816" +checksum = "5ef32d3854235cea74e7abdd294ed2bc2a9398e18ddf36a762d51a5b41d9a7b9" dependencies = [ "itertools", "oxc_allocator", @@ -2907,9 +2987,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51599c317d4e19c07978bde4af8a40844a2410fb7b455836c3441a41865acfa" +checksum = "e755e211d32f1d52255eebb39dfb8a787620e5d163e3c6bbe3ad8cb0cca75c40" dependencies = [ "compact_str", "oxc-miette", @@ -2921,9 +3001,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee16107642e60a1f53dac2dbaaf1a1cdc696c0f7427946f8b8906f3e09a85f0" +checksum = "817dd63f7421c08f9cf1fe1463208590cc0cca524bfafa3f252ae519587cdf70" dependencies = [ "bitflags 2.9.1", "cow-utils", @@ -2943,14 +3023,14 @@ dependencies = [ [[package]] name = "oxc_transformer" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd95eedb44dd519dfe157334512977de744725cda45d4c9005369e3f155d6b4" +checksum = "82d36761790513485a774a582a7d0b7e4e1c86f3f0dc76bb776363b367fb2482" dependencies = [ "base64 0.22.1", "compact_str", "cow-utils", - "indexmap 2.9.0", + "indexmap 2.10.0", "itoa", "memchr", "oxc-browserslist", @@ -2974,9 +3054,9 @@ dependencies = [ [[package]] name = "oxc_transformer_plugins" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa2124adf31739ae94ca9ee5ecb727b42ad7bc3e33ca072a6c9fa791d9f7b5" +checksum = "3a5e4e99e6ca788dc45d5216e8311c49957c29c8d521fa5d82887b316856cb3c" dependencies = [ "cow-utils", "itoa", @@ -2996,9 +3076,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.73.2" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd6b3996c515cf7ce1c204b3b01a5a1469462412885ea6fa7f0d236d8f2c454" +checksum = "c80b098496732c9cdcb34398332c9a7679cfc20d6258a6cad56e4031c889db32" dependencies = [ "itoa", "oxc_allocator", @@ -3014,9 +3094,9 @@ dependencies = [ [[package]] name = "papaya" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af228bb1296c9b044ee75e2a2325409c2d899bcfcc6150e5e41f148e0a87dd20" +checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f" dependencies = [ "equivalent", "seize", @@ -3070,6 +3150,18 @@ dependencies = [ "once_cell", ] +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -3094,35 +3186,36 @@ checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset", "hashbrown 0.15.4", - "indexmap 2.9.0", + "indexmap 2.10.0", "serde", ] [[package]] name = "phf" -version = "0.11.3" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ "phf_macros", "phf_shared", + "serde", ] [[package]] name = "phf_generator" -version = "0.11.3" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +checksum = "2cbb1126afed61dd6368748dae63b1ee7dc480191c6262a3b4ff1e29d86a6c5b" dependencies = [ + "fastrand", "phf_shared", - "rand 0.8.5", ] [[package]] name = "phf_macros" -version = "0.11.3" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +checksum = "d713258393a82f091ead52047ca779d37e5766226d009de21696c4e667044368" dependencies = [ "phf_generator", "phf_shared", @@ -3133,9 +3226,9 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.3" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" dependencies = [ "siphasher", ] @@ -3178,6 +3271,26 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "pnp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6da10bf83fe58dfcd4ab14ab45c8d3a375663cfd8f44e924c54379b431dbd3" +dependencies = [ + "byteorder", + "clean-path", + "concurrent_lru", + "fancy-regex", + "miniz_oxide", + "path-slash", + "pathdiff", + "radix_trie", + "rustc-hash", + "serde", + "serde_json", + "thiserror 2.0.12", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -3199,6 +3312,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -3264,6 +3383,16 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.5" @@ -3424,9 +3553,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "regress" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366" +checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010" dependencies = [ "hashbrown 0.15.4", "memchr", @@ -3563,9 +3692,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe8c9d1c68d67dd9f97ecbc6f932b60eb289c5dbddd8aa1405484a8fd2fcd984" +checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" dependencies = [ "dyn-clone", "ref-cast", @@ -3576,9 +3705,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ca9fcb757952f8e8629b9ab066fc62da523c46c2b247b1708a3be06dd82530b" +checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80" dependencies = [ "proc-macro2", "quote", @@ -3691,7 +3820,7 @@ version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "itoa", "memchr", "ryu", @@ -3710,9 +3839,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.9" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" dependencies = [ "serde", ] @@ -3731,16 +3860,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf65a400f8f66fb7b0552869ad70157166676db75ed8181f8104ea91cf9d0b42" +checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.9.0", + "indexmap 2.10.0", "schemars 0.9.0", + "schemars 1.0.4", "serde", "serde_derive", "serde_json", @@ -3750,9 +3880,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81679d9ed988d5e9a5e6531dc3f2c28efbd639cbd1dfb628df08edea6004da77" +checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ "darling", "proc-macro2", @@ -4057,15 +4187,17 @@ dependencies = [ [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "pin-project-lite", + "slab", "socket2", "tokio-macros", "windows-sys 0.52.0", @@ -4108,44 +4240,42 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.23" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac" dependencies = [ + "indexmap 2.10.0", "serde", "serde_spanned", "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.9.0", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", + "toml_parser", + "toml_writer", "winnow", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "toml_datetime" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_parser" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" [[package]] name = "tower" @@ -4444,9 +4574,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vfs" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ec343ec20aa715908fd028a4b8e7c99a349d13143224222e4d61c316d1e7f0a" +checksum = "9e723b9e1c02a3cf9f9d0de6a4ddb8cdc1df859078902fe0ae0589d615711ae6" dependencies = [ "filetime", ] @@ -4799,12 +4929,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" -dependencies = [ - "memchr", -] +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" [[package]] name = "wit-bindgen-rt" @@ -4869,9 +4996,9 @@ dependencies = [ [[package]] name = "zip" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ab361742de920c5535880f89bbd611ee62002bf11341d16a5f057bb8ba6899" +checksum = "9aed4ac33e8eb078c89e6cbb1d5c4c7703ec6d299fc3e7c3695af8f8b423468b" dependencies = [ "aes", "arbitrary", @@ -4882,10 +5009,11 @@ dependencies = [ "flate2", "getrandom 0.3.3", "hmac", - "indexmap 2.9.0", + "indexmap 2.10.0", "liblzma", "memchr", "pbkdf2", + "ppmd-rust", "sha1", "time", "zeroize", diff --git a/Cargo.toml b/Cargo.toml index da2139dcf..4013b4726 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" package.edition = "2024" -package.version = "0.0.71" +package.version = "0.0.80" package.homepage = "https://bitcoinresearchkit.org" package.repository = "https://github.com/bitcoinresearchkit/brk" package.readme = "README.md" @@ -14,6 +14,10 @@ lto = "fat" codegen-units = 1 panic = "abort" +[profile.profiling] +inherits = "release" +debug = true + [profile.dist] inherits = "release" @@ -23,40 +27,39 @@ axum = "0.8.4" bincode = { version = "2.0.1", features = ["serde"] } bitcoin = { version = "0.32.6", features = ["serde"] } bitcoincore-rpc = "0.19.0" -brk_bundler = { version = "0.0.71", path = "crates/brk_bundler" } -brk_cli = { version = "0.0.71", path = "crates/brk_cli" } -brk_computer = { version = "0.0.71", path = "crates/brk_computer" } -brk_core = { version = "0.0.71", path = "crates/brk_core" } -brk_exit = { version = "0.0.71", path = "crates/brk_exit" } -brk_fetcher = { version = "0.0.71", path = "crates/brk_fetcher" } -brk_indexer = { version = "0.0.71", path = "crates/brk_indexer" } -brk_interface = { version = "0.0.71", path = "crates/brk_interface" } -brk_logger = { version = "0.0.71", path = "crates/brk_logger" } -brk_mcp = { version = "0.0.71", path = "crates/brk_mcp" } -brk_parser = { version = "0.0.71", path = "crates/brk_parser" } -brk_rmcp = { version = "0.1.8", features = ["transport-streamable-http-server", "transport-worker"]} +brk_bundler = { version = "0.0.80", path = "crates/brk_bundler" } +brk_cli = { version = "0.0.80", path = "crates/brk_cli" } +brk_computer = { version = "0.0.80", path = "crates/brk_computer" } +brk_core = { version = "0.0.80", path = "crates/brk_core" } +brk_exit = { version = "0.0.80", path = "crates/brk_exit" } +brk_fetcher = { version = "0.0.80", path = "crates/brk_fetcher" } +brk_indexer = { version = "0.0.80", path = "crates/brk_indexer" } +brk_interface = { version = "0.0.80", path = "crates/brk_interface" } +brk_logger = { version = "0.0.80", path = "crates/brk_logger" } +brk_mcp = { version = "0.0.80", path = "crates/brk_mcp" } +brk_parser = { version = "0.0.80", path = "crates/brk_parser" } +brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]} # brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]} -brk_server = { version = "0.0.71", path = "crates/brk_server" } -brk_state = { version = "0.0.71", path = "crates/brk_state" } -brk_store = { version = "0.0.71", path = "crates/brk_store" } -brk_vec = { version = "0.0.71", path = "crates/brk_vec" } +brk_server = { version = "0.0.80", path = "crates/brk_server" } +brk_store = { version = "0.0.80", path = "crates/brk_store" } +brk_vec = { version = "0.0.80", path = "crates/brk_vec" } byteview = "=0.6.1" -clap = { version = "4.5.40", features = ["string"] } -clap_derive = "4.5.40" +clap = { version = "4.5.41", features = ["string"] } +clap_derive = "4.5.41" color-eyre = "0.6.5" derive_deref = "1.1.1" fjall = "2.11.1" jiff = "0.2.15" log = { version = "0.4.27" } -minreq = { version = "2.13.4", features = ["https", "serde_json"] } +minreq = { version = "2.14.0", features = ["https", "serde_json"] } rayon = "1.10.0" -schemars = "1.0.1" +schemars = "1.0.4" serde = { version = "1.0.219" } serde_bytes = "0.11.17" serde_derive = "1.0.219" serde_json = { version = "1.0.140", features = ["float_roundtrip"] } tabled = "0.20.0" -tokio = { version = "1.45.1", features = ["rt-multi-thread"] } +tokio = { version = "1.46.1", features = ["rt-multi-thread"] } zerocopy = { version = "0.8.26" } zerocopy-derive = "0.8.26" diff --git a/README.md b/README.md index 72620cf74..66dc9ddbc 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible Just like the website, it's entirely free, with no authentication or rate-limiting. - **[AI](https://github.com/bitcoinresearchkit/brk/blob/main/crates/brk_mcp/README.md#brk-mcp)** \ LLMs have to possibility to connect to BRK's backend through a [MCP](https://modelcontextprotocol.io/introduction). \ - It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts. \ - One-shot output examples: [Document](https://claude.ai/public/artifacts/71194d29-f965-417c-ba09-fdf0e4ecb1d5) // [Dashboard](https://claude.ai/public/artifacts/beef143f-399a-4ed4-b8bf-c986b776de42) // [Dashboard 2](https://claude.ai/public/artifacts/5430ae49-bb3d-4fc1-ab24-f1e33deb40dc) + It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts. - **[CLI](https://crates.io/crates/brk_cli)** \ Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \ The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end. @@ -73,7 +72,6 @@ In contrast, existing alternatives tend to be either [very costly](https://studi - [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust - [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer` -- [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) - [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec - [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/) diff --git a/TODO.md b/TODO.md new file mode 100644 index 000000000..8ae918195 --- /dev/null +++ b/TODO.md @@ -0,0 +1,99 @@ +# TODO + +- __crates__ + - _cli_ + - check disk space on first launch + - add custom path support for config.toml + - maybe add bitcoind download and launch support + - via: https://github.com/rust-bitcoin/corepc/blob/master/node + - test read/write speed, add warning if too low (<2gb/s) + - pull latest version and notify is out of date + - _computer_ + - **add rollback of states (in stateful)** + - add support for per index computation + - fix feerate which is always ZERO due to coinbase transaction + - before computing multiple sources check their length, panic if not equal + - add oracle price dataset (https://utxo.live/oracle/UTXOracle.py) + - add address counts relative to all datasets + - make decade, quarter, year datasets `computed` instead of `eager` + - add 6 months (semester) interval datasets to builder + - some datasets in `indexes` can probably be removed + - add revived/sent supply datasets + - add `in-sats` version of all price datasets (average and co) + - add `p2pk` group (sum of `p2pk33` and `p2pk65`) + - add chopiness datasets + - add utxo count, address count, supply data for by reused addresses in groups by address type + - add more date ranges (3-6 months and more) + - add puell multiple dataset + - add pi cycle dataset + - add emas of price + - add 7d and 30d ema to sell side risk ratio and sopr + - don't compute everything for all cohorts as some datasets combinations are irrelevant + - addresses/utxos by amount don't need mvrvz for example + - add all possible charts from: + - https://mainnet.observer + - https://glassnode.com + - https://checkonchain.com + - https://researchbitcoin.net/exciting-update-coming-to-the-bitcoin-lab/ + - https://mempool.space/research + - _indexer_ + - parse only the needed block number + - maybe using https://developer.bitcoin.org/reference/rpc/getblockhash.html + - _interface_ + - create pagination enum + - from to + - from option + - to option + - page + option default 1000 max 1000 + - from/to/count params don’t cap all combinations + - example: from -10,000 count 10, won’t work if underlying vec isn’t 10k or more long + - _parser_ + - save `vec` file instead of `json` + - support lock file, process in read only if already opened in write mode + - if less than X (10 maybe ?) get block using rpc instead of parsing the block files + - _server_ + - api + - add extensions support (.json .csv …) + - if format instead of extension then don't download file + - add support for https (rustls) + - _vec_ + - add native lock file support (once it's available in stable rust) + - improve compressed mode (slow reads) + - add ema support +- __docs__ + - _README_ + - add a comparison table with alternatives + - add contribution section where help is needed + - documentation/mcp/datasets/different front ends + - add faq +- __websites__ + - _default_ + - explorer + - blocks + - transactions + - addresses + - miners + - maybe xpubs + - charts + - improve some names and colors + - remove `sum` series when it's a duplicate of the `base` (in subsidy for example) + - selected unit sometimes changes when going back end forth + - add support for custom charts + - separate z-score charts from "realized price" (with their own prices), have 4y, 2y and 1y + - price scale format depends on unit, hide digits for sats for example (if/when possible) + - table + - pagination + - exports (.json, .csv,…) + - search + - datasets add legend, and keywords ? + - height/address/txid + - api + - add api page with interactivity + - global + - **fix navigation/history** + - move share button to footer ? + - Use `ichart.createPane()` in wrapper + - improve behavior when local storage is unavailable + - by having a global state +- __global__ + - check `TODO`s in codebase diff --git a/assets/claude-step3.png b/assets/claude-step3.png index 3aa5fdc3f..af3735423 100644 Binary files a/assets/claude-step3.png and b/assets/claude-step3.png differ diff --git a/assets/claude-step4.png b/assets/claude-step4.png index 65e1aa519..17277ec61 100644 Binary files a/assets/claude-step4.png and b/assets/claude-step4.png differ diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index 99b24ae24..4d872f4b9 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -21,7 +21,6 @@ full = [ "parser", "interface", "server", - "state", "store", "vec", ] @@ -36,7 +35,6 @@ mcp = ["brk_mcp"] parser = ["brk_parser"] interface = ["brk_interface"] server = ["brk_server"] -state = ["brk_state"] store = ["brk_store"] vec = ["brk_vec"] @@ -53,7 +51,6 @@ brk_mcp = { workspace = true, optional = true } brk_parser = { workspace = true, optional = true } brk_interface = { workspace = true, optional = true } brk_server = { workspace = true, optional = true } -brk_state = { workspace = true, optional = true } brk_store = { workspace = true, optional = true } brk_vec = { workspace = true, optional = true } diff --git a/crates/brk/flamegraph.sh b/crates/brk/flamegraph.sh new file mode 100755 index 000000000..1a00c11d7 --- /dev/null +++ b/crates/brk/flamegraph.sh @@ -0,0 +1,2 @@ +cargo build --profile profiling +flamegraph -- ../../target/profiling/brk diff --git a/crates/brk/samply.sh b/crates/brk/samply.sh new file mode 100755 index 000000000..f11387b4f --- /dev/null +++ b/crates/brk/samply.sh @@ -0,0 +1,2 @@ +cargo build --profile profiling +samply record ../../target/profiling/brk diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs index 12223db75..f568a5ba6 100644 --- a/crates/brk/src/lib.rs +++ b/crates/brk/src/lib.rs @@ -47,10 +47,6 @@ pub use brk_interface as interface; #[doc(inline)] pub use brk_server as server; -#[cfg(feature = "state")] -#[doc(inline)] -pub use brk_state as state; - #[cfg(feature = "store")] #[doc(inline)] pub use brk_store as store; diff --git a/crates/brk_bundler/Cargo.toml b/crates/brk_bundler/Cargo.toml index 0b4986718..f3f6ebb68 100644 --- a/crates/brk_bundler/Cargo.toml +++ b/crates/brk_bundler/Cargo.toml @@ -9,7 +9,8 @@ repository.workspace = true [dependencies] log = { workspace = true } -notify = "8.0.0" -brk_rolldown = "0.0.1" +notify = "8.1.0" +brk_rolldown = "0.1.1" +# brk_rolldown = { path = "../../../rolldown/crates/rolldown"} sugar_path = "1.2.0" tokio = { workspace = true } diff --git a/crates/brk_bundler/src/lib.rs b/crates/brk_bundler/src/lib.rs index 7da9996ef..c86a16872 100644 --- a/crates/brk_bundler/src/lib.rs +++ b/crates/brk_bundler/src/lib.rs @@ -81,7 +81,7 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i let mut entry_watcher = notify::recommended_watcher( move |res: Result| match res { Ok(_) => write_index_clone(), - Err(e) => error!("watch error: {:?}", e), + Err(e) => error!("watch error: {e:?}"), }, ) .unwrap(); @@ -112,7 +112,7 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i let _ = fs::copy(&source_path, &dist_path); } }), - Err(e) => error!("watch error: {:?}", e), + Err(e) => error!("watch error: {e:?}"), }, ) .unwrap(); diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 475097f47..c3b74a84e 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -24,7 +24,7 @@ color-eyre = { workspace = true } log = { workspace = true } serde = { workspace = true } tokio = { workspace = true } -toml = "0.8.23" +toml = "0.9.2" [[bin]] name = "brk" diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 132b60f33..32cc7b581 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -8,6 +8,7 @@ homepage.workspace = true repository.workspace = true [dependencies] +bincode = { workspace = true } bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } brk_core = { workspace = true } @@ -16,10 +17,17 @@ brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } -brk_state = { workspace = true } +brk_store = { workspace = true } brk_vec = { workspace = true } color-eyre = { workspace = true } +derive_deref = { workspace = true } +either = "1.15.0" fjall = { workspace = true } -jiff = { workspace = true } log = { workspace = true } rayon = { workspace = true } +serde = { workspace = true } +zerocopy = { workspace = true } +zerocopy-derive = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["zerocopy"] diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index d7520f530..ee69e3f57 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -13,7 +13,8 @@ pub fn main() -> color_eyre::Result<()> { brk_logger::init(Some(Path::new(".log"))); - let bitcoin_dir = default_bitcoin_path(); + // let bitcoin_dir = default_bitcoin_path(); + let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin"); let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( "http://localhost:8332", @@ -23,11 +24,11 @@ pub fn main() -> color_eyre::Result<()> { // Can't increase main thread's stack programatically, thus we need to use another thread thread::Builder::new() - .stack_size(32 * 1024 * 1024) + .stack_size(256 * 1024 * 1024) .spawn(move || -> color_eyre::Result<()> { let parser = Parser::new(bitcoin_dir.join("blocks"), default_brk_path(), rpc); - let _outputs_dir = default_brk_path().join("outputs"); + let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs"); let outputs_dir = _outputs_dir.as_path(); // let outputs_dir = Path::new("../../_outputs"); diff --git a/crates/brk_computer/flamegraph.sh b/crates/brk_computer/flamegraph.sh new file mode 100755 index 000000000..23a23b08b --- /dev/null +++ b/crates/brk_computer/flamegraph.sh @@ -0,0 +1,2 @@ +cargo build --profile profiling +flamegraph -- ../../target/profiling/examples/main diff --git a/crates/brk_computer/samply.sh b/crates/brk_computer/samply.sh new file mode 100755 index 000000000..d32802d12 --- /dev/null +++ b/crates/brk_computer/samply.sh @@ -0,0 +1,2 @@ +cargo build --example main --profile profiling +samply record ../../target/profiling/examples/main diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 6bb159a1e..c2e5b1cbc 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -10,12 +10,14 @@ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_vec::{Computation, Format}; +use log::info; +mod states; mod stores; mod utils; mod vecs; -use log::info; +use states::*; use stores::Stores; use vecs::Vecs; @@ -66,7 +68,12 @@ impl Computer { exit: &Exit, ) -> color_eyre::Result<()> { info!("Computing..."); - self.vecs - .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit) + self.vecs.compute( + indexer, + starting_indexes, + self.fetcher.as_mut(), + exit, + &mut self.stores, + ) } } diff --git a/crates/brk_state/src/block.rs b/crates/brk_computer/src/states/block.rs similarity index 100% rename from crates/brk_state/src/block.rs rename to crates/brk_computer/src/states/block.rs diff --git a/crates/brk_computer/src/states/cohorts/address.rs b/crates/brk_computer/src/states/cohorts/address.rs new file mode 100644 index 000000000..41e5a3749 --- /dev/null +++ b/crates/brk_computer/src/states/cohorts/address.rs @@ -0,0 +1,120 @@ +use std::path::Path; + +use brk_core::{AddressData, Dollars, Height, Result, Sats}; + +use crate::SupplyState; + +use super::CohortState; + +#[derive(Clone)] +pub struct AddressCohortState { + pub address_count: usize, + pub inner: CohortState, +} + +impl AddressCohortState { + pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result { + Ok(Self { + address_count: 0, + inner: CohortState::default_and_import(path, name, compute_dollars)?, + }) + } + + pub fn height(&self) -> Option { + self.inner.height() + } + + pub fn reset_price_to_amount(&mut self) -> Result<()> { + self.inner.reset_price_to_amount() + } + + pub fn reset_single_iteration_values(&mut self) { + self.inner.reset_single_iteration_values(); + } + + #[allow(clippy::too_many_arguments)] + pub fn send( + &mut self, + addressdata: &mut AddressData, + value: Sats, + current_price: Option, + prev_price: Option, + blocks_old: usize, + days_old: f64, + older_than_hour: bool, + ) -> Result<()> { + let compute_price = current_price.is_some(); + + let prev_realized_price = compute_price.then(|| addressdata.realized_price()); + let prev_supply_state = SupplyState { + utxos: addressdata.outputs_len as usize, + value: addressdata.amount(), + }; + + addressdata.send(value, prev_price)?; + + let supply_state = SupplyState { + utxos: addressdata.outputs_len as usize, + value: addressdata.amount(), + }; + + self.inner.send_( + &SupplyState { utxos: 1, value }, + current_price, + prev_price, + blocks_old, + days_old, + older_than_hour, + compute_price.then(|| (addressdata.realized_price(), &supply_state)), + prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)), + ); + + Ok(()) + } + + pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option) { + let compute_price = price.is_some(); + + let prev_realized_price = compute_price.then(|| address_data.realized_price()); + let prev_supply_state = SupplyState { + utxos: address_data.outputs_len as usize, + value: address_data.amount(), + }; + + address_data.receive(value, price); + + let supply_state = SupplyState { + utxos: address_data.outputs_len as usize, + value: address_data.amount(), + }; + + self.inner.receive_( + &SupplyState { utxos: 1, value }, + price, + compute_price.then(|| (address_data.realized_price(), &supply_state)), + prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)), + ); + } + + pub fn add(&mut self, addressdata: &AddressData) { + self.address_count += 1; + self.inner.increment_( + &addressdata.into(), + addressdata.realized_cap, + addressdata.realized_price(), + ); + } + + pub fn subtract(&mut self, addressdata: &AddressData) { + self.address_count = self.address_count.checked_sub(1).unwrap(); + self.inner.decrement_( + &addressdata.into(), + addressdata.realized_cap, + addressdata.realized_price(), + ); + } + + pub fn commit(&mut self, height: Height) -> Result<()> { + self.inner.commit(height) + } +} diff --git a/crates/brk_state/src/cohort.rs b/crates/brk_computer/src/states/cohorts/common.rs similarity index 59% rename from crates/brk_state/src/cohort.rs rename to crates/brk_computer/src/states/cohorts/common.rs index dde7d5435..ec63241d6 100644 --- a/crates/brk_state/src/cohort.rs +++ b/crates/brk_computer/src/states/cohorts/common.rs @@ -2,9 +2,7 @@ use std::{cmp::Ordering, path::Path}; use brk_core::{CheckedSub, Dollars, Height, Result, Sats}; -use crate::{PriceToAmount, UnrealizedState}; - -use super::{RealizedState, SupplyState}; +use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState}; #[derive(Clone)] pub struct CohortState { @@ -12,7 +10,8 @@ pub struct CohortState { pub realized: Option, pub satblocks_destroyed: Sats, pub satdays_destroyed: Sats, - pub price_to_amount: PriceToAmount, + + price_to_amount: PriceToAmount, } impl CohortState { @@ -26,6 +25,22 @@ impl CohortState { }) } + pub fn height(&self) -> Option { + self.price_to_amount.height() + } + + pub fn reset_price_to_amount(&mut self) -> Result<()> { + self.price_to_amount.reset() + } + + pub fn price_to_amount_first_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.price_to_amount.first_key_value() + } + + pub fn price_to_amount_last_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.price_to_amount.last_key_value() + } + pub fn reset_single_iteration_values(&mut self) { self.satdays_destroyed = Sats::ZERO; self.satblocks_destroyed = Sats::ZERO; @@ -41,7 +56,23 @@ impl CohortState { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.increment(supply_state, price); - *self.price_to_amount.entry(price).or_default() += supply_state.value; + self.price_to_amount.increment(price, supply_state); + } + } + } + + pub fn increment_( + &mut self, + supply_state: &SupplyState, + realized_cap: Dollars, + realized_price: Dollars, + ) { + self.supply += supply_state; + + if supply_state.value > Sats::ZERO { + if let Some(realized) = self.realized.as_mut() { + realized.increment_(realized_cap); + self.price_to_amount.increment(realized_price, supply_state); } } } @@ -53,27 +84,60 @@ impl CohortState { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.decrement(supply_state, price); - self.decrement_price_to_amount(supply_state, price); + self.price_to_amount.decrement(price, supply_state); } } } - fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars) { - let amount = self.price_to_amount.get_mut(&price).unwrap(); - *amount -= supply_state.value; - if *amount == Sats::ZERO { - self.price_to_amount.remove(&price); + pub fn decrement_( + &mut self, + supply_state: &SupplyState, + realized_cap: Dollars, + realized_price: Dollars, + ) { + self.supply -= supply_state; + + if supply_state.value > Sats::ZERO { + if let Some(realized) = self.realized.as_mut() { + realized.decrement_(realized_cap); + self.price_to_amount.decrement(realized_price, supply_state); + } } } pub fn receive(&mut self, supply_state: &SupplyState, price: Option) { + self.receive_( + supply_state, + price, + price.map(|price| (price, supply_state)), + None, + ); + } + + pub fn receive_( + &mut self, + supply_state: &SupplyState, + price: Option, + price_to_amount_increment: Option<(Dollars, &SupplyState)>, + price_to_amount_decrement: Option<(Dollars, &SupplyState)>, + ) { self.supply += supply_state; if supply_state.value > Sats::ZERO { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.receive(supply_state, price); - *self.price_to_amount.entry(price).or_default() += supply_state.value; + + if let Some((price, supply)) = price_to_amount_increment + && supply.value.is_not_zero() + { + self.price_to_amount.increment(price, supply); + } + if let Some((price, supply)) = price_to_amount_decrement + && supply.value.is_not_zero() + { + self.price_to_amount.decrement(price, supply); + } } } } @@ -87,6 +151,34 @@ impl CohortState { days_old: f64, older_than_hour: bool, ) { + self.send_( + supply_state, + current_price, + prev_price, + blocks_old, + days_old, + older_than_hour, + None, + prev_price.map(|prev_price| (prev_price, supply_state)), + ); + } + + #[allow(clippy::too_many_arguments)] + pub fn send_( + &mut self, + supply_state: &SupplyState, + current_price: Option, + prev_price: Option, + blocks_old: usize, + days_old: f64, + older_than_hour: bool, + price_to_amount_increment: Option<(Dollars, &SupplyState)>, + price_to_amount_decrement: Option<(Dollars, &SupplyState)>, + ) { + if supply_state.utxos == 0 { + return; + } + self.supply -= supply_state; if supply_state.value > Sats::ZERO { @@ -99,7 +191,16 @@ impl CohortState { let current_price = current_price.unwrap(); let prev_price = prev_price.unwrap(); realized.send(supply_state, current_price, prev_price, older_than_hour); - self.decrement_price_to_amount(supply_state, prev_price); + if let Some((price, supply)) = price_to_amount_increment + && supply.value.is_not_zero() + { + self.price_to_amount.increment(price, supply); + } + if let Some((price, supply)) = price_to_amount_decrement + && supply.value.is_not_zero() + { + self.price_to_amount.decrement(price, supply); + } } } } diff --git a/crates/brk_computer/src/states/cohorts/mod.rs b/crates/brk_computer/src/states/cohorts/mod.rs new file mode 100644 index 000000000..b3dc7b22f --- /dev/null +++ b/crates/brk_computer/src/states/cohorts/mod.rs @@ -0,0 +1,7 @@ +mod address; +mod common; +mod utxo; + +pub use address::*; +pub use common::*; +pub use utxo::*; diff --git a/crates/brk_computer/src/states/cohorts/utxo.rs b/crates/brk_computer/src/states/cohorts/utxo.rs new file mode 100644 index 000000000..15ccd2836 --- /dev/null +++ b/crates/brk_computer/src/states/cohorts/utxo.rs @@ -0,0 +1,27 @@ +use std::path::Path; + +use brk_core::{Height, Result}; +use derive_deref::{Deref, DerefMut}; + +use super::CohortState; + +#[derive(Clone, Deref, DerefMut)] +pub struct UTXOCohortState(CohortState); + +impl UTXOCohortState { + pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result { + Ok(Self(CohortState::default_and_import( + path, + name, + compute_dollars, + )?)) + } + + pub fn height(&self) -> Option { + self.0.height() + } + + pub fn reset_price_to_amount(&mut self) -> Result<()> { + self.0.reset_price_to_amount() + } +} diff --git a/crates/brk_computer/src/states/mod.rs b/crates/brk_computer/src/states/mod.rs new file mode 100644 index 000000000..5976a47b9 --- /dev/null +++ b/crates/brk_computer/src/states/mod.rs @@ -0,0 +1,15 @@ +mod block; +mod cohorts; +mod price_to_amount; +mod realized; +mod supply; +mod transacted; +mod unrealized; + +pub use block::*; +pub use cohorts::*; +pub use price_to_amount::*; +pub use realized::*; +pub use supply::*; +pub use transacted::*; +pub use unrealized::*; diff --git a/crates/brk_state/src/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs similarity index 74% rename from crates/brk_state/src/price_to_amount.rs rename to crates/brk_computer/src/states/price_to_amount.rs index 1b68adca5..493b76336 100644 --- a/crates/brk_state/src/price_to_amount.rs +++ b/crates/brk_computer/src/states/price_to_amount.rs @@ -2,7 +2,6 @@ use std::{ collections::BTreeMap, fs::{self, File}, io::{BufReader, BufWriter}, - ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; @@ -11,6 +10,8 @@ use brk_core::{Dollars, Height, Result, Sats}; use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; +use crate::states::SupplyState; + #[derive(Clone, Debug)] pub struct PriceToAmount { pathbuf: PathBuf, @@ -46,8 +47,36 @@ impl PriceToAmount { }) } + pub fn iter(&self) -> impl Iterator { + self.state.iter() + } + + pub fn is_empty(&self) -> bool { + self.state.is_empty() + } + + pub fn first_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.state.first_key_value() + } + + pub fn last_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.state.last_key_value() + } + + pub fn increment(&mut self, price: Dollars, supply_state: &SupplyState) { + *self.state.entry(price).or_default() += supply_state.value; + } + + pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) { + let amount = self.state.get_mut(&price).unwrap(); + *amount -= supply_state.value; + if *amount == Sats::ZERO { + self.state.remove(&price); + } + } + pub fn reset(&mut self) -> Result<()> { - self.clear(); + self.state.clear(); self.height = None; fs::remove_dir_all(&self.pathbuf)?; fs::create_dir_all(&self.pathbuf)?; @@ -89,16 +118,3 @@ impl PriceToAmount { path.join("height") } } - -impl Deref for PriceToAmount { - type Target = BTreeMap; - fn deref(&self) -> &Self::Target { - &self.state - } -} - -impl DerefMut for PriceToAmount { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.state - } -} diff --git a/crates/brk_state/src/realized.rs b/crates/brk_computer/src/states/realized.rs similarity index 89% rename from crates/brk_state/src/realized.rs rename to crates/brk_computer/src/states/realized.rs index 7d673ae8d..a4eea6f66 100644 --- a/crates/brk_state/src/realized.rs +++ b/crates/brk_computer/src/states/realized.rs @@ -42,6 +42,10 @@ impl RealizedState { return; } + self.increment_(price * supply_state.value) + } + + pub fn increment_(&mut self, realized_cap: Dollars) { if self.cap == Dollars::NAN { self.cap = Dollars::ZERO; self.profit = Dollars::ZERO; @@ -52,13 +56,15 @@ impl RealizedState { self.adj_value_destroyed = Dollars::ZERO; } - let value = price * supply_state.value; - self.cap += value; + self.cap += realized_cap; } pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) { - let value = price * supply_state.value; - self.cap = self.cap.checked_sub(value).unwrap(); + self.decrement_(price * supply_state.value); + } + + pub fn decrement_(&mut self, realized_cap: Dollars) { + self.cap = self.cap.checked_sub(realized_cap).unwrap(); } pub fn receive(&mut self, supply_state: &SupplyState, current_price: Dollars) { diff --git a/crates/brk_state/src/supply.rs b/crates/brk_computer/src/states/supply.rs similarity index 80% rename from crates/brk_state/src/supply.rs rename to crates/brk_computer/src/states/supply.rs index c9bd95442..4c26830d3 100644 --- a/crates/brk_state/src/supply.rs +++ b/crates/brk_computer/src/states/supply.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, SubAssign}; -use brk_core::{CheckedSub, Sats}; +use brk_core::{AddressData, CheckedSub, Sats}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -39,3 +39,12 @@ impl SubAssign<&SupplyState> for SupplyState { self.value = self.value.checked_sub(rhs.value).unwrap(); } } + +impl From<&AddressData> for SupplyState { + fn from(value: &AddressData) -> Self { + Self { + utxos: value.outputs_len as usize, + value: value.amount(), + } + } +} diff --git a/crates/brk_computer/src/states/transacted.rs b/crates/brk_computer/src/states/transacted.rs new file mode 100644 index 000000000..64f8c3b73 --- /dev/null +++ b/crates/brk_computer/src/states/transacted.rs @@ -0,0 +1,48 @@ +use std::ops::{Add, AddAssign}; + +use brk_core::{ByAmountRange, GroupedByType, OutputType, Sats}; + +use super::SupplyState; + +#[derive(Default, Debug)] +pub struct Transacted { + pub spendable_supply: SupplyState, + pub by_type: GroupedByType, + pub by_size_group: ByAmountRange, +} + +impl Transacted { + #[allow(clippy::inconsistent_digit_grouping)] + pub fn iterate(&mut self, value: Sats, _type: OutputType) { + let supply = SupplyState { utxos: 1, value }; + + *self.by_type.get_mut(_type) += &supply; + + if _type.is_unspendable() { + return; + } + + self.spendable_supply += &supply; + + *self.by_size_group.get_mut(value) += &supply; + } +} + +impl Add for Transacted { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self { + spendable_supply: self.spendable_supply + rhs.spendable_supply, + by_type: self.by_type + rhs.by_type, + by_size_group: self.by_size_group + rhs.by_size_group, + } + } +} + +impl AddAssign for Transacted { + fn add_assign(&mut self, rhs: Self) { + self.by_size_group += rhs.by_size_group; + self.spendable_supply += &rhs.spendable_supply; + self.by_type += rhs.by_type; + } +} diff --git a/crates/brk_state/src/unrealized.rs b/crates/brk_computer/src/states/unrealized.rs similarity index 100% rename from crates/brk_state/src/unrealized.rs rename to crates/brk_computer/src/states/unrealized.rs diff --git a/crates/brk_computer/src/stores.rs b/crates/brk_computer/src/stores.rs index b1b886c68..f52d94ac1 100644 --- a/crates/brk_computer/src/stores.rs +++ b/crates/brk_computer/src/stores.rs @@ -1,34 +1,641 @@ -use std::path::Path; +use std::{path::Path, thread}; -use brk_core::Version; -use fjall::TransactionalKeyspace; +use brk_core::{ + AddressData, ByAddressType, EmptyAddressData, Height, OutputType, P2AAddressIndex, + P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, + P2WPKHAddressIndex, P2WSHAddressIndex, Result, TypeIndex, Version, +}; +use brk_store::{AnyStore, Store}; +use fjall::{PersistMode, TransactionalKeyspace}; +use log::info; -const _VERSION: Version = Version::ZERO; +use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource}; + +const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Stores { - // pub address_to_utxos_received: Store, - // pub address_to_utxos_spent: Store, + keyspace: TransactionalKeyspace, + + pub p2aaddressindex_to_addressdata: Store, + pub p2aaddressindex_to_emptyaddressdata: Store, + pub p2pk33addressindex_to_addressdata: Store, + pub p2pk33addressindex_to_emptyaddressdata: Store, + pub p2pk65addressindex_to_addressdata: Store, + pub p2pk65addressindex_to_emptyaddressdata: Store, + pub p2pkhaddressindex_to_addressdata: Store, + pub p2pkhaddressindex_to_emptyaddressdata: Store, + pub p2shaddressindex_to_addressdata: Store, + pub p2shaddressindex_to_emptyaddressdata: Store, + pub p2traddressindex_to_addressdata: Store, + pub p2traddressindex_to_emptyaddressdata: Store, + pub p2wpkhaddressindex_to_addressdata: Store, + pub p2wpkhaddressindex_to_emptyaddressdata: Store, + pub p2wshaddressindex_to_addressdata: Store, + pub p2wshaddressindex_to_emptyaddressdata: Store, } impl Stores { - pub fn import(_: &Path, _: Version, _: &TransactionalKeyspace) -> color_eyre::Result { - // let address_to_utxos_received = Store::import( - // keyspace.clone(), - // path, - // "address_to_utxos_received", - // version + VERSION + Version::ZERO, - // )?; - // let address_to_utxos_spent = Store::import( - // keyspace.clone(), - // path, - // "address_to_utxos_spent", - // version + VERSION + Version::ZERO, - // )?; + pub fn import( + path: &Path, + version: Version, + keyspace: &TransactionalKeyspace, + ) -> color_eyre::Result { + let ( + (p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata), + (p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata), + (p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata), + (p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata), + (p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata), + (p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata), + (p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata), + (p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata), + ) = thread::scope(|scope| { + let p2a = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2aaddressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2aaddressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + let p2pk33 = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2pk33addressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2pk33addressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + let p2pk65 = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2pk65addressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2pk65addressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + let p2pkh = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2pkhaddressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2pkhaddressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + let p2sh = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2shaddressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2shaddressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + let p2tr = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2traddressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2traddressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + let p2wpkh = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2wpkhaddressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2wpkhaddressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + let p2wsh = scope.spawn(|| { + ( + Store::import( + keyspace, + path, + "p2wshaddressindex_to_addressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + Store::import( + keyspace, + path, + "p2wshaddressindex_to_emptyaddressdata", + version + VERSION + Version::ZERO, + None, + ) + .unwrap(), + ) + }); + + ( + p2a.join().unwrap(), + p2pk33.join().unwrap(), + p2pk65.join().unwrap(), + p2pkh.join().unwrap(), + p2sh.join().unwrap(), + p2tr.join().unwrap(), + p2wpkh.join().unwrap(), + p2wsh.join().unwrap(), + ) + }); Ok(Self { - // address_to_utxos_received, - // address_to_utxos_spent, + keyspace: keyspace.clone(), + + p2aaddressindex_to_addressdata, + p2aaddressindex_to_emptyaddressdata, + + p2pk33addressindex_to_addressdata, + p2pk33addressindex_to_emptyaddressdata, + + p2pk65addressindex_to_addressdata, + p2pk65addressindex_to_emptyaddressdata, + + p2pkhaddressindex_to_addressdata, + p2pkhaddressindex_to_emptyaddressdata, + + p2shaddressindex_to_addressdata, + p2shaddressindex_to_emptyaddressdata, + + p2traddressindex_to_addressdata, + p2traddressindex_to_emptyaddressdata, + + p2wpkhaddressindex_to_addressdata, + p2wpkhaddressindex_to_emptyaddressdata, + + p2wshaddressindex_to_addressdata, + p2wshaddressindex_to_emptyaddressdata, }) } + + pub fn starting_height(&self) -> Height { + self.as_slice() + .into_iter() + .map(|store| store.height().map(Height::incremented).unwrap_or_default()) + .min() + .unwrap() + } + + pub fn reset(&mut self) -> Result<()> { + info!("Resetting stores..."); + info!("> If it gets stuck here, stop the program and start it again"); + + self.as_mut_slice() + .into_iter() + .try_for_each(|store| store.reset())?; + + self.keyspace + .persist(PersistMode::SyncAll) + .map_err(|e| e.into()) + } + + pub fn get_addressdata( + &self, + address_type: OutputType, + type_index: TypeIndex, + ) -> Result> { + Ok(match address_type { + OutputType::P2A => self + .p2aaddressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2PK33 => self + .p2pk33addressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2PK65 => self + .p2pk65addressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2PKH => self + .p2pkhaddressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2SH => self + .p2shaddressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2TR => self + .p2traddressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2WPKH => self + .p2wpkhaddressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2WSH => self + .p2wshaddressindex_to_addressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + _ => unreachable!(), + }) + } + + pub fn get_emptyaddressdata( + &self, + address_type: OutputType, + type_index: TypeIndex, + ) -> Result> { + Ok(match address_type { + OutputType::P2A => self + .p2aaddressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2PK33 => self + .p2pk33addressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2PK65 => self + .p2pk65addressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2PKH => self + .p2pkhaddressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2SH => self + .p2shaddressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2TR => self + .p2traddressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2WPKH => self + .p2wpkhaddressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + OutputType::P2WSH => self + .p2wshaddressindex_to_emptyaddressdata + .get(&type_index.into())? + .map(|c| c.into_owned()), + _ => unreachable!(), + }) + } + + pub fn commit( + &mut self, + height: Height, + addresstype_to_typeindex_to_addressdata: AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + ) -> Result<()> { + let ByAddressType { + p2pk65, + p2pk33, + p2pkh, + p2sh, + p2wpkh, + p2wsh, + p2tr, + p2a, + } = addresstype_to_typeindex_to_addressdata.unwrap(); + + let ByAddressType { + p2pk65: empty_p2pk65, + p2pk33: empty_p2pk33, + p2pkh: empty_p2pkh, + p2sh: empty_p2sh, + p2wpkh: empty_p2wpkh, + p2wsh: empty_p2wsh, + p2tr: empty_p2tr, + p2a: empty_p2a, + } = addresstype_to_typeindex_to_emptyaddressdata.unwrap(); + + thread::scope(|s| { + s.spawn(|| { + self.p2aaddressindex_to_addressdata.commit_( + height, + empty_p2a + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2a.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + s.spawn(|| { + self.p2pk33addressindex_to_addressdata.commit_( + height, + empty_p2pk33 + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2pk33.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + s.spawn(|| { + self.p2pk65addressindex_to_addressdata.commit_( + height, + empty_p2pk65 + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2pk65.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + s.spawn(|| { + self.p2pkhaddressindex_to_addressdata.commit_( + height, + empty_p2pkh + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2pkh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + s.spawn(|| { + self.p2shaddressindex_to_addressdata.commit_( + height, + empty_p2sh + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2sh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + s.spawn(|| { + self.p2traddressindex_to_addressdata.commit_( + height, + empty_p2tr + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2tr.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + s.spawn(|| { + self.p2wpkhaddressindex_to_addressdata.commit_( + height, + empty_p2wpkh + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2wpkh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + s.spawn(|| { + self.p2wshaddressindex_to_addressdata.commit_( + height, + empty_p2wsh + .iter() + .filter(|(_, addressdata)| addressdata.is_from_addressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + p2wsh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + }); + + thread::scope(|scope| { + scope.spawn(|| { + self.p2aaddressindex_to_emptyaddressdata.commit_( + height, + p2a.iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2a.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + scope.spawn(|| { + self.p2pk33addressindex_to_emptyaddressdata.commit_( + height, + p2pk33 + .iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2pk33.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + scope.spawn(|| { + self.p2pk65addressindex_to_emptyaddressdata.commit_( + height, + p2pk65 + .iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2pk65.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + scope.spawn(|| { + self.p2pkhaddressindex_to_emptyaddressdata.commit_( + height, + p2pkh + .iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2pkh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + scope.spawn(|| { + self.p2shaddressindex_to_emptyaddressdata.commit_( + height, + p2sh.iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2sh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + scope.spawn(|| { + self.p2traddressindex_to_emptyaddressdata.commit_( + height, + p2tr.iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2tr.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + scope.spawn(|| { + self.p2wpkhaddressindex_to_emptyaddressdata.commit_( + height, + p2wpkh + .iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2wpkh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + scope.spawn(|| { + self.p2wshaddressindex_to_emptyaddressdata.commit_( + height, + p2wsh + .iter() + .filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata()) + .map(|(typeindex, _)| (*typeindex).into()), + empty_p2wsh.iter().map(|(typeindex, addressdata)| { + ((*typeindex).into(), addressdata.deref().clone()) + }), + ) + }); + }); + + self.keyspace + .persist(PersistMode::SyncAll) + .map_err(|e| e.into()) + } + + pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] { + [ + &self.p2aaddressindex_to_addressdata, + &self.p2aaddressindex_to_emptyaddressdata, + &self.p2pk33addressindex_to_addressdata, + &self.p2pk33addressindex_to_emptyaddressdata, + &self.p2pk65addressindex_to_addressdata, + &self.p2pk65addressindex_to_emptyaddressdata, + &self.p2pkhaddressindex_to_addressdata, + &self.p2pkhaddressindex_to_emptyaddressdata, + &self.p2shaddressindex_to_addressdata, + &self.p2shaddressindex_to_emptyaddressdata, + &self.p2traddressindex_to_addressdata, + &self.p2traddressindex_to_emptyaddressdata, + &self.p2wpkhaddressindex_to_addressdata, + &self.p2wpkhaddressindex_to_emptyaddressdata, + &self.p2wshaddressindex_to_addressdata, + &self.p2wshaddressindex_to_emptyaddressdata, + ] + } + + fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 16] { + [ + &mut self.p2aaddressindex_to_addressdata, + &mut self.p2aaddressindex_to_emptyaddressdata, + &mut self.p2pk33addressindex_to_addressdata, + &mut self.p2pk33addressindex_to_emptyaddressdata, + &mut self.p2pk65addressindex_to_addressdata, + &mut self.p2pk65addressindex_to_emptyaddressdata, + &mut self.p2pkhaddressindex_to_addressdata, + &mut self.p2pkhaddressindex_to_emptyaddressdata, + &mut self.p2shaddressindex_to_addressdata, + &mut self.p2shaddressindex_to_emptyaddressdata, + &mut self.p2traddressindex_to_addressdata, + &mut self.p2traddressindex_to_emptyaddressdata, + &mut self.p2wpkhaddressindex_to_addressdata, + &mut self.p2wpkhaddressindex_to_emptyaddressdata, + &mut self.p2wshaddressindex_to_addressdata, + &mut self.p2wshaddressindex_to_emptyaddressdata, + ] + } } diff --git a/crates/brk_computer/src/vecs/blocks.rs b/crates/brk_computer/src/vecs/blocks.rs index 75644192d..9a3a0da59 100644 --- a/crates/brk_computer/src/vecs/blocks.rs +++ b/crates/brk_computer/src/vecs/blocks.rs @@ -8,9 +8,11 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format}; +use crate::vecs::grouped::Source; + use super::{ Indexes, - grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions}, + grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, VecBuilderOptions}, indexes, }; @@ -34,8 +36,9 @@ impl Vecs { pub fn forced_import( path: &Path, version: Version, - _computation: Computation, + computation: Computation, format: Format, + indexes: &indexes::Vecs, ) -> color_eyre::Result { Ok(Self { height_to_interval: EagerVec::forced_import( @@ -47,18 +50,22 @@ impl Vecs { timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import( path, "timestamp", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_first(), + computation, + indexes, + VecBuilderOptions::default().add_first(), )?, indexes_to_block_interval: ComputedVecsFromHeight::forced_import( path, "block_interval", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_percentiles() .add_minmax() .add_average(), @@ -66,32 +73,32 @@ impl Vecs { indexes_to_block_count: ComputedVecsFromHeight::forced_import( path, "block_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_weight: ComputedVecsFromHeight::forced_import( path, "block_weight", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_size: ComputedVecsFromHeight::forced_import( path, "block_size", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, height_to_vbytes: EagerVec::forced_import( path, @@ -102,12 +109,12 @@ impl Vecs { indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( path, "block_vbytes", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, difficultyepoch_to_timestamp: EagerVec::forced_import( path, diff --git a/crates/brk_computer/src/vecs/cointime.rs b/crates/brk_computer/src/vecs/cointime.rs index ecb6563ef..c98e410e7 100644 --- a/crates/brk_computer/src/vecs/cointime.rs +++ b/crates/brk_computer/src/vecs/cointime.rs @@ -7,13 +7,13 @@ use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; use crate::vecs::{ fetched, - grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight}, + grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, Source}, stateful, transactions, }; use super::{ Indexes, - grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions}, + grouped::{ComputedVecsFromHeight, VecBuilderOptions}, indexes, }; @@ -51,8 +51,9 @@ impl Vecs { pub fn forced_import( path: &Path, version: Version, - _computation: Computation, + computation: Computation, format: Format, + indexes: &indexes::Vecs, fetched: Option<&fetched::Vecs>, ) -> color_eyre::Result { let compute_dollars = fetched.is_some(); @@ -61,194 +62,234 @@ impl Vecs { indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import( path, "coinblocks_created", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import( path, "coinblocks_stored", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_liveliness: ComputedVecsFromHeight::forced_import( path, "liveliness", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_vaultedness: ComputedVecsFromHeight::forced_import( path, "vaultedness", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import( path, "activity_to_vaultedness_ratio", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import( path, "vaulted_supply", - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, )?, indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import( path, "active_supply", - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, )?, indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import( path, "thermo_cap", - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_investor_cap: ComputedVecsFromHeight::forced_import( path, "investor_cap", - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import( path, "vaulted_cap", - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_active_cap: ComputedVecsFromHeight::forced_import( path, "active_cap", - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import( path, "vaulted_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( path, "vaulted_price", - false, + Source::None, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_active_price: ComputedVecsFromHeight::forced_import( path, "active_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( path, "active_price", - false, + Source::None, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import( path, "true_market_mean", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import( path, "true_market_mean", - false, + Source::None, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import( path, "cointime_value_destroyed", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import( path, "cointime_value_created", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import( path, "cointime_value_stored", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_price: ComputedVecsFromHeight::forced_import( path, "cointime_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import( path, "cointime_cap", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( path, "cointime_price", - false, + Source::None, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, }) } @@ -264,7 +305,7 @@ impl Vecs { stateful: &stateful::Vecs, exit: &Exit, ) -> color_eyre::Result<()> { - let circulating_supply = &stateful.utxos_vecs.all.1.height_to_supply; + let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply; self.indexes_to_coinblocks_created.compute_all( indexer, @@ -282,7 +323,7 @@ impl Vecs { )?; let indexes_to_coinblocks_destroyed = - &stateful.utxos_vecs.all.1.indexes_to_coinblocks_destroyed; + &stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed; self.indexes_to_coinblocks_stored.compute_all( indexer, @@ -392,7 +433,7 @@ impl Vecs { if let Some(fetched) = fetched { let realized_cap = stateful - .utxos_vecs + .utxo_vecs .all .1 .height_to_realized_cap @@ -400,7 +441,7 @@ impl Vecs { .unwrap(); let realized_price = stateful - .utxos_vecs + .utxo_vecs .all .1 .indexes_to_realized_price diff --git a/crates/brk_computer/src/vecs/constants.rs b/crates/brk_computer/src/vecs/constants.rs index 2180f727d..6607df3d5 100644 --- a/crates/brk_computer/src/vecs/constants.rs +++ b/crates/brk_computer/src/vecs/constants.rs @@ -5,9 +5,11 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format}; +use crate::vecs::grouped::Source; + use super::{ Indexes, - grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions}, + grouped::{ComputedVecsFromHeight, VecBuilderOptions}, indexes, }; @@ -25,41 +27,50 @@ impl Vecs { pub fn forced_import( path: &Path, version: Version, - _computation: Computation, + computation: Computation, format: Format, + indexes: &indexes::Vecs, ) -> color_eyre::Result { Ok(Self { constant_0: ComputedVecsFromHeight::forced_import( path, "constant_0", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, constant_1: ComputedVecsFromHeight::forced_import( path, "constant_1", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, constant_50: ComputedVecsFromHeight::forced_import( path, "constant_50", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, constant_100: ComputedVecsFromHeight::forced_import( path, "constant_100", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, }) } diff --git a/crates/brk_computer/src/vecs/fetched.rs b/crates/brk_computer/src/vecs/fetched.rs index 2fdd97e17..8ae500870 100644 --- a/crates/brk_computer/src/vecs/fetched.rs +++ b/crates/brk_computer/src/vecs/fetched.rs @@ -2,18 +2,22 @@ use std::path::Path; use brk_core::{ Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex, - OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, Version, WeekIndex, YearIndex, + OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, + YearIndex, }; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format, StoredIndex}; +use brk_vec::{ + AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, StoredIndex, + VecIterator, +}; + +use crate::vecs::grouped::Source; use super::{ Indexes, - grouped::{ - ComputedVecsFromDateIndex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions, - }, + grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeightStrict, VecBuilderOptions}, indexes, }; @@ -57,6 +61,8 @@ pub struct Vecs { pub monthindex_to_ohlc_in_sats: EagerVec, pub quarterindex_to_ohlc: EagerVec, pub quarterindex_to_ohlc_in_sats: EagerVec, + pub semesterindex_to_ohlc: EagerVec, + pub semesterindex_to_ohlc_in_sats: EagerVec, pub yearindex_to_ohlc: EagerVec, pub yearindex_to_ohlc_in_sats: EagerVec, // pub halvingepoch_to_ohlc: StorableVec, @@ -72,8 +78,9 @@ impl Vecs { pub fn forced_import( path: &Path, version: Version, - _computation: Computation, + computation: Computation, format: Format, + indexes: &indexes::Vecs, ) -> color_eyre::Result { let mut fetched_path = path.to_owned(); fetched_path.pop(); @@ -83,7 +90,7 @@ impl Vecs { dateindex_to_ohlc_in_cents: EagerVec::forced_import( &fetched_path, "ohlc_in_cents", - version + VERSION + Version::ZERO, + version + Version::ZERO, format, )?, dateindex_to_ohlc: EagerVec::forced_import( @@ -125,7 +132,7 @@ impl Vecs { height_to_ohlc_in_cents: EagerVec::forced_import( &fetched_path, "ohlc_in_cents", - version + VERSION + Version::ZERO, + version + Version::ZERO, format, )?, height_to_ohlc: EagerVec::forced_import( @@ -167,122 +174,138 @@ impl Vecs { timeindexes_to_open: ComputedVecsFromDateIndex::forced_import( path, "open", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_first(), + computation, + indexes, + VecBuilderOptions::default().add_first(), )?, timeindexes_to_high: ComputedVecsFromDateIndex::forced_import( path, "high", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_max(), + computation, + indexes, + VecBuilderOptions::default().add_max(), )?, timeindexes_to_low: ComputedVecsFromDateIndex::forced_import( path, "low", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_min(), + computation, + indexes, + VecBuilderOptions::default().add_min(), )?, timeindexes_to_close: ComputedVecsFromDateIndex::forced_import( path, "close", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import( path, "open_in_sats", - true, + Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_first(), + computation, + indexes, + VecBuilderOptions::default().add_first(), )?, timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import( path, "high_in_sats", - true, + Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_max(), + computation, + indexes, + VecBuilderOptions::default().add_max(), )?, timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import( path, "low_in_sats", - true, + Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_min(), + computation, + indexes, + VecBuilderOptions::default().add_min(), )?, timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import( path, "close_in_sats", - true, + Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( path, "open", version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_first(), + VecBuilderOptions::default().add_first(), )?, chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( path, "high", version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_max(), + VecBuilderOptions::default().add_max(), )?, chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( path, "low", version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_min(), + VecBuilderOptions::default().add_min(), )?, chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( path, "close", version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + VecBuilderOptions::default().add_last(), )?, chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import( path, "open_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_first(), + VecBuilderOptions::default().add_first(), )?, chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import( path, "high_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_max(), + VecBuilderOptions::default().add_max(), )?, chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import( path, "low_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_min(), + VecBuilderOptions::default().add_min(), )?, chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import( path, "close_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + VecBuilderOptions::default().add_last(), )?, weekindex_to_ohlc: EagerVec::forced_import( path, @@ -332,6 +355,18 @@ impl Vecs { version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, + semesterindex_to_ohlc: EagerVec::forced_import( + path, + "ohlc", + version + VERSION + Version::ZERO, + format, + )?, + semesterindex_to_ohlc_in_sats: EagerVec::forced_import( + path, + "ohlc_in_sats", + version + VERSION + VERSION_IN_SATS + Version::ZERO, + format, + )?, yearindex_to_ohlc: EagerVec::forced_import( path, "ohlc", @@ -422,21 +457,34 @@ impl Vecs { exit, )?; + let mut prev = None; self.dateindex_to_ohlc_in_cents.compute_transform( starting_indexes.dateindex, &indexes.dateindex_to_date, |(di, d, this)| { - let mut ohlc = fetcher.get_date(d).unwrap(); - if let Some(prev) = di.decremented() { - let prev_open = *this - .get_or_read(prev, &this.mmap().load()) - .unwrap() - .unwrap() - .close; + if prev.is_none() { + let i = di.unwrap_to_usize(); + prev.replace(if i > 0 { + this.into_iter().unwrap_get_inner_(i - 1) + } else { + OHLCCents::default() + }); + } + + let ohlc = if di.unwrap_to_usize() + 100 >= this.len() + && let Ok(mut ohlc) = fetcher.get_date(d) + { + let prev_open = *prev.as_ref().unwrap().close; *ohlc.open = prev_open; *ohlc.high = (*ohlc.high).max(prev_open); *ohlc.low = (*ohlc.low).min(prev_open); - } + ohlc + } else { + prev.clone().unwrap() + }; + + prev.replace(ohlc.clone()); + (di, ohlc) }, exit, @@ -604,12 +652,15 @@ impl Vecs { starting_indexes.weekindex, self.timeindexes_to_close.weekindex.unwrap_last(), |(i, close, ..)| { + let open = weekindex_first_iter.unwrap_get_inner(i); + let high = weekindex_max_iter.unwrap_get_inner(i); + let low = weekindex_min_iter.unwrap_get_inner(i); ( i, OHLCDollars { - open: weekindex_first_iter.unwrap_get_inner(i), - high: weekindex_max_iter.unwrap_get_inner(i), - low: weekindex_min_iter.unwrap_get_inner(i), + open, + high, + low, close, }, ) @@ -687,6 +738,27 @@ impl Vecs { exit, )?; + let mut semesterindex_first_iter = + self.timeindexes_to_open.semesterindex.unwrap_first().iter(); + let mut semesterindex_max_iter = self.timeindexes_to_high.semesterindex.unwrap_max().iter(); + let mut semesterindex_min_iter = self.timeindexes_to_low.semesterindex.unwrap_min().iter(); + self.semesterindex_to_ohlc.compute_transform( + starting_indexes.semesterindex, + self.timeindexes_to_close.semesterindex.unwrap_last(), + |(i, close, ..)| { + ( + i, + OHLCDollars { + open: semesterindex_first_iter.unwrap_get_inner(i), + high: semesterindex_max_iter.unwrap_get_inner(i), + low: semesterindex_min_iter.unwrap_get_inner(i), + close, + }, + ) + }, + exit, + )?; + let mut yearindex_first_iter = self.timeindexes_to_open.yearindex.unwrap_first().iter(); let mut yearindex_max_iter = self.timeindexes_to_high.yearindex.unwrap_max().iter(); let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter(); @@ -1038,6 +1110,40 @@ impl Vecs { exit, )?; + let mut semesterindex_first_iter = self + .timeindexes_to_open_in_sats + .semesterindex + .unwrap_first() + .iter(); + let mut semesterindex_max_iter = self + .timeindexes_to_high_in_sats + .semesterindex + .unwrap_max() + .iter(); + let mut semesterindex_min_iter = self + .timeindexes_to_low_in_sats + .semesterindex + .unwrap_min() + .iter(); + self.semesterindex_to_ohlc_in_sats.compute_transform( + starting_indexes.semesterindex, + self.timeindexes_to_close_in_sats + .semesterindex + .unwrap_last(), + |(i, close, ..)| { + ( + i, + OHLCSats { + open: semesterindex_first_iter.unwrap_get_inner(i), + high: semesterindex_max_iter.unwrap_get_inner(i), + low: semesterindex_min_iter.unwrap_get_inner(i), + close, + }, + ) + }, + exit, + )?; + let mut yearindex_first_iter = self .timeindexes_to_open_in_sats .yearindex @@ -1127,6 +1233,7 @@ impl Vecs { &self.difficultyepoch_to_ohlc, &self.monthindex_to_ohlc, &self.quarterindex_to_ohlc, + &self.semesterindex_to_ohlc, &self.yearindex_to_ohlc, // &self.halvingepoch_to_ohlc, &self.decadeindex_to_ohlc, @@ -1136,6 +1243,7 @@ impl Vecs { &self.difficultyepoch_to_ohlc_in_sats, &self.monthindex_to_ohlc_in_sats, &self.quarterindex_to_ohlc_in_sats, + &self.semesterindex_to_ohlc_in_sats, &self.yearindex_to_ohlc_in_sats, // &self.halvingepoch_to_ohlc_in_sats, &self.decadeindex_to_ohlc_in_sats, diff --git a/crates/brk_computer/src/vecs/grouped/builder_computed.rs b/crates/brk_computer/src/vecs/grouped/builder_computed.rs new file mode 100644 index 000000000..8115570c9 --- /dev/null +++ b/crates/brk_computer/src/vecs/grouped/builder_computed.rs @@ -0,0 +1,474 @@ +use std::path::Path; + +use brk_core::{FromCoarserIndex, Result, Version}; +use brk_exit::Exit; +use brk_vec::{ + AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation, + ComputedVec, ComputedVecFrom2, Format, StoredIndex, +}; + +use crate::vecs::grouped::{EagerVecBuilder, VecBuilderOptions}; + +use super::ComputedType; + +#[allow(clippy::type_complexity)] +#[derive(Clone)] +pub struct ComputedVecBuilder +where + I: StoredIndex, + T: ComputedType, + S2T: ComputedType, +{ + pub first: Option>>, + pub average: Option>>, + pub sum: Option>>, + pub max: Option>>, + pub min: Option>>, + pub last: Option>>, + pub cumulative: Option>>, +} + +const VERSION: Version = Version::ZERO; + +impl ComputedVecBuilder +where + I: StoredIndex, + T: ComputedType + 'static, + S1I: StoredIndex + 'static + FromCoarserIndex, + S2T: ComputedType, +{ + #[allow(clippy::too_many_arguments)] + pub fn forced_import( + path: &Path, + name: &str, + version: Version, + format: Format, + computation: Computation, + source: Option>, + source_extra: &EagerVecBuilder, + len_source: BoxedAnyIterableVec, + options: ComputedVecBuilderOptions, + ) -> color_eyre::Result { + let only_one_active = options.is_only_one_active(); + + let suffix = |s: &str| format!("{name}_{s}"); + + let maybe_suffix = |s: &str| { + if only_one_active { + name.to_string() + } else { + suffix(s) + } + }; + + Ok(Self { + first: options.first.then(|| { + Box::new( + ComputedVec::forced_import_or_init_from_2( + computation, + path, + &maybe_suffix("first"), + version + VERSION + Version::ZERO, + format, + source_extra + .first + .as_ref() + .map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()), + len_source.clone(), + |i: I, source, len_source| { + if i.unwrap_to_usize() >= len_source.len() { + return None; + } + source + .next_at(S1I::min_from(i)) + .map(|(_, cow)| cow.into_owned()) + }, + ) + .unwrap(), + ) + }), + last: options.last.then(|| { + Box::new( + ComputedVec::forced_import_or_init_from_2( + computation, + path, + name, + version + VERSION + Version::ZERO, + format, + source_extra.last.as_ref().map_or_else( + || { + source + .as_ref() + .unwrap_or_else(|| { + dbg!(path, name, I::to_string()); + panic!() + }) + .clone() + }, + |v| v.clone(), + ), + len_source.clone(), + |i: I, source, len_source| { + if i.unwrap_to_usize() >= len_source.len() { + return None; + } + source + .next_at(S1I::max_from(i, source.len())) + .map(|(_, cow)| cow.into_owned()) + }, + ) + .unwrap(), + ) + }), + min: options.min.then(|| { + Box::new( + ComputedVec::forced_import_or_init_from_2( + computation, + path, + &maybe_suffix("min"), + version + VERSION + Version::ZERO, + format, + source_extra + .min + .as_ref() + .map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()), + len_source.clone(), + |i: I, source, len_source| { + if i.unwrap_to_usize() >= len_source.len() { + return None; + } + S1I::inclusive_range_from(i, source.len()) + .flat_map(|i| source.next_at(i).map(|(_, cow)| cow.into_owned())) + .min() + }, + ) + .unwrap(), + ) + }), + max: options.max.then(|| { + Box::new( + ComputedVec::forced_import_or_init_from_2( + computation, + path, + &maybe_suffix("max"), + version + VERSION + Version::ZERO, + format, + source_extra + .max + .as_ref() + .map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()), + len_source.clone(), + |i: I, source, len_source| { + if i.unwrap_to_usize() >= len_source.len() { + return None; + } + S1I::inclusive_range_from(i, source.len()) + .flat_map(|i| source.next_at(i).map(|(_, cow)| cow.into_owned())) + .max() + }, + ) + .unwrap(), + ) + }), + average: options.average.then(|| { + Box::new( + ComputedVec::forced_import_or_init_from_2( + computation, + path, + &maybe_suffix("average"), + version + VERSION + Version::ZERO, + format, + source_extra + .average + .as_ref() + .map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()), + len_source.clone(), + |i: I, source, len_source| { + if i.unwrap_to_usize() >= len_source.len() { + return None; + } + let vec = S1I::inclusive_range_from(i, source.len()) + .flat_map(|i| source.next_at(i).map(|(_, cow)| cow.into_owned())) + .collect::>(); + if vec.is_empty() { + return None; + } + let mut sum = T::from(0); + let len = vec.len(); + vec.into_iter().for_each(|v| sum += v); + Some(sum / len) + }, + ) + .unwrap(), + ) + }), + sum: options.sum.then(|| { + Box::new( + ComputedVec::forced_import_or_init_from_2( + computation, + path, + &(if !options.last && !options.average && !options.min && !options.max { + name.to_string() + } else { + maybe_suffix("sum") + }), + version + VERSION + Version::ZERO, + format, + source_extra + .sum + .as_ref() + .map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()), + len_source.clone(), + |i: I, source, len_source| { + if i.unwrap_to_usize() >= len_source.len() { + return None; + } + let vec = S1I::inclusive_range_from(i, source.len()) + .flat_map(|i| source.next_at(i).map(|(_, cow)| cow.into_owned())) + .collect::>(); + if vec.is_empty() { + return None; + } + let mut sum = T::from(0); + vec.into_iter().for_each(|v| sum += v); + Some(sum) + }, + ) + .unwrap(), + ) + }), + cumulative: options.cumulative.then(|| { + Box::new( + ComputedVec::forced_import_or_init_from_2( + computation, + path, + &suffix("cumulative"), + version + VERSION + Version::ZERO, + format, + source_extra.cumulative.as_ref().unwrap().boxed_clone(), + len_source.clone(), + |i: I, source, len_source| { + if i.unwrap_to_usize() >= len_source.len() { + return None; + } + source + .next_at(S1I::max_from(i, source.len())) + .map(|(_, cow)| cow.into_owned()) + }, + ) + .unwrap(), + ) + }), + }) + } + + pub fn compute_if_necessary( + &mut self, + max_from: I, + len_source: &impl AnyIterableVec, + exit: &Exit, + ) -> Result<()> { + if let Some(first) = self.first.as_mut() { + first.compute_if_necessary(max_from, len_source, exit)?; + } + if let Some(last) = self.last.as_mut() { + last.compute_if_necessary(max_from, len_source, exit)?; + } + if let Some(min) = self.min.as_mut() { + min.compute_if_necessary(max_from, len_source, exit)?; + } + if let Some(max) = self.max.as_mut() { + max.compute_if_necessary(max_from, len_source, exit)?; + } + if let Some(average) = self.average.as_mut() { + average.compute_if_necessary(max_from, len_source, exit)?; + } + if let Some(sum) = self.sum.as_mut() { + sum.compute_if_necessary(max_from, len_source, exit)?; + } + if let Some(cumulative) = self.cumulative.as_mut() { + cumulative.compute_if_necessary(max_from, len_source, exit)?; + } + + Ok(()) + } + + pub fn starting_index(&self, max_from: I) -> I { + max_from.min(I::from( + self.vecs().into_iter().map(|v| v.len()).min().unwrap(), + )) + } + + pub fn unwrap_first(&self) -> &ComputedVecFrom2 { + self.first.as_ref().unwrap() + } + #[allow(unused)] + pub fn unwrap_average(&self) -> &ComputedVecFrom2 { + self.average.as_ref().unwrap() + } + pub fn unwrap_sum(&self) -> &ComputedVecFrom2 { + self.sum.as_ref().unwrap() + } + pub fn unwrap_max(&self) -> &ComputedVecFrom2 { + self.max.as_ref().unwrap() + } + pub fn unwrap_min(&self) -> &ComputedVecFrom2 { + self.min.as_ref().unwrap() + } + pub fn unwrap_last(&self) -> &ComputedVecFrom2 { + self.last.as_ref().unwrap() + } + #[allow(unused)] + pub fn unwrap_cumulative(&self) -> &ComputedVecFrom2 { + self.cumulative.as_ref().unwrap() + } + + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + let mut v: Vec<&dyn AnyCollectableVec> = vec![]; + + if let Some(first) = self.first.as_ref() { + v.push(first.as_ref()); + } + if let Some(last) = self.last.as_ref() { + v.push(last.as_ref()); + } + if let Some(min) = self.min.as_ref() { + v.push(min.as_ref()); + } + if let Some(max) = self.max.as_ref() { + v.push(max.as_ref()); + } + if let Some(average) = self.average.as_ref() { + v.push(average.as_ref()); + } + if let Some(sum) = self.sum.as_ref() { + v.push(sum.as_ref()); + } + if let Some(cumulative) = self.cumulative.as_ref() { + v.push(cumulative.as_ref()); + } + + v + } +} + +#[derive(Default, Clone, Copy)] +pub struct ComputedVecBuilderOptions { + average: bool, + sum: bool, + max: bool, + min: bool, + first: bool, + last: bool, + cumulative: bool, +} + +impl From for ComputedVecBuilderOptions { + fn from(value: VecBuilderOptions) -> Self { + Self { + average: value.average(), + sum: value.sum(), + max: value.max(), + min: value.min(), + first: value.first(), + last: value.last(), + cumulative: value.cumulative(), + } + } +} + +impl ComputedVecBuilderOptions { + pub fn add_first(mut self) -> Self { + self.first = true; + self + } + + pub fn add_last(mut self) -> Self { + self.last = true; + self + } + + pub fn add_min(mut self) -> Self { + self.min = true; + self + } + + pub fn add_max(mut self) -> Self { + self.max = true; + self + } + + pub fn add_average(mut self) -> Self { + self.average = true; + self + } + + pub fn add_sum(mut self) -> Self { + self.sum = true; + self + } + + pub fn add_cumulative(mut self) -> Self { + self.cumulative = true; + self + } + + #[allow(unused)] + pub fn rm_min(mut self) -> Self { + self.min = false; + self + } + + #[allow(unused)] + pub fn rm_max(mut self) -> Self { + self.max = false; + self + } + + #[allow(unused)] + pub fn rm_average(mut self) -> Self { + self.average = false; + self + } + + #[allow(unused)] + pub fn rm_sum(mut self) -> Self { + self.sum = false; + self + } + + #[allow(unused)] + pub fn rm_cumulative(mut self) -> Self { + self.cumulative = false; + self + } + + pub fn add_minmax(mut self) -> Self { + self.min = true; + self.max = true; + self + } + + pub fn is_only_one_active(&self) -> bool { + [ + self.average, + self.sum, + self.max, + self.min, + self.first, + self.last, + self.cumulative, + ] + .iter() + .filter(|b| **b) + .count() + == 1 + } + + pub fn copy_self_extra(&self) -> Self { + Self { + cumulative: self.cumulative, + ..Self::default() + } + } +} diff --git a/crates/brk_computer/src/vecs/grouped/builder.rs b/crates/brk_computer/src/vecs/grouped/builder_eager.rs similarity index 94% rename from crates/brk_computer/src/vecs/grouped/builder.rs rename to crates/brk_computer/src/vecs/grouped/builder_eager.rs index 88eb692b6..3a7d6e536 100644 --- a/crates/brk_computer/src/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/vecs/grouped/builder_eager.rs @@ -10,7 +10,7 @@ use crate::utils::get_percentile; use super::ComputedType; #[derive(Clone, Debug)] -pub struct ComputedVecBuilder +pub struct EagerVecBuilder where I: StoredIndex, T: ComputedType, @@ -31,7 +31,7 @@ where const VERSION: Version = Version::ZERO; -impl ComputedVecBuilder +impl EagerVecBuilder where I: StoredIndex, T: ComputedType, @@ -41,20 +41,10 @@ where name: &str, version: Version, format: Format, - options: StorableVecGeneatorOptions, + options: VecBuilderOptions, ) -> color_eyre::Result { let only_one_active = options.is_only_one_active(); - let prefix = |s: &str| format!("{s}_{name}"); - - let maybe_prefix = |s: &str| { - if only_one_active { - name.to_string() - } else { - prefix(s) - } - }; - let suffix = |s: &str| format!("{name}_{s}"); let maybe_suffix = |s: &str| { @@ -70,7 +60,7 @@ where Box::new( EagerVec::forced_import( path, - &maybe_prefix("first"), + &maybe_suffix("first"), version + VERSION + Version::ZERO, format, ) @@ -130,7 +120,7 @@ where Box::new( EagerVec::forced_import( path, - &(if !options.last { + &(if !options.last && !options.average && !options.min && !options.max { name.to_string() } else { maybe_suffix("sum") @@ -145,7 +135,7 @@ where Box::new( EagerVec::forced_import( path, - &prefix("cumulative"), + &suffix("cumulative"), version + VERSION + Version::ZERO, format, ) @@ -221,7 +211,7 @@ where cumulative_vec.iter().unwrap_get_inner(index) }); source.iter_at(index).try_for_each(|(i, v)| -> Result<()> { - cumulative = cumulative.clone() + v.into_inner(); + cumulative = cumulative.clone() + v.into_owned(); cumulative_vec.forced_push_at(i, cumulative.clone(), exit) })?; @@ -261,7 +251,7 @@ where first_indexes .iter_at(index) .try_for_each(|(i, first_index)| -> Result<()> { - let first_index = first_index.into_inner(); + let first_index = first_index.into_owned(); let count_index = count_indexes_iter.unwrap_get_inner(i); @@ -284,7 +274,7 @@ where // dbg!(first_index, count_index, last_index); // }) // .unwrap() - // .into_inner(); + // .into_owned(); last.forced_push_at(index, v, exit)?; } @@ -304,7 +294,7 @@ where source_iter.set(first_index); let mut values = (&mut source_iter) .take(*count_index) - .map(|(_, v)| v.into_inner()) + .map(|(_, v)| v.into_owned()) .collect::>(); if needs_sorted { @@ -394,7 +384,7 @@ where pub fn from_aligned( &mut self, max_from: I, - source: &ComputedVecBuilder, + source: &EagerVecBuilder, first_indexes: &impl AnyIterableVec, count_indexes: &impl AnyIterableVec, exit: &Exit, @@ -435,7 +425,7 @@ where first_indexes .iter_at(index) .try_for_each(|(i, first_index, ..)| -> Result<()> { - let first_index = first_index.into_inner(); + let first_index = first_index.into_owned(); let count_index = count_indexes_iter.unwrap_get_inner(i); @@ -473,7 +463,7 @@ where source_max_iter.set(first_index); let mut values = source_max_iter .take(*count_index) - .map(|(_, v)| v.into_inner()) + .map(|(_, v)| v.into_owned()) .collect::>(); values.sort_unstable(); max.forced_push_at(i, values.last().unwrap().clone(), exit)?; @@ -484,7 +474,7 @@ where source_min_iter.set(first_index); let mut values = source_min_iter .take(*count_index) - .map(|(_, v)| v.into_inner()) + .map(|(_, v)| v.into_owned()) .collect::>(); values.sort_unstable(); min.forced_push_at(i, values.first().unwrap().clone(), exit)?; @@ -497,7 +487,7 @@ where source_average_iter.set(first_index); let values = source_average_iter .take(*count_index) - .map(|(_, v)| v.into_inner()) + .map(|(_, v)| v.into_owned()) .collect::>(); let len = values.len(); @@ -513,7 +503,7 @@ where source_sum_iter.set(first_index); let values = source_sum_iter .take(*count_index) - .map(|(_, v)| v.into_inner()) + .map(|(_, v)| v.into_owned()) .collect::>(); let sum = values.into_iter().fold(T::from(0), |a, b| a + b); @@ -716,7 +706,7 @@ where } #[derive(Default, Clone, Copy)] -pub struct StorableVecGeneatorOptions { +pub struct VecBuilderOptions { average: bool, sum: bool, max: bool, @@ -731,7 +721,55 @@ pub struct StorableVecGeneatorOptions { cumulative: bool, } -impl StorableVecGeneatorOptions { +impl VecBuilderOptions { + pub fn average(&self) -> bool { + self.average + } + + pub fn sum(&self) -> bool { + self.sum + } + + pub fn max(&self) -> bool { + self.max + } + + pub fn _90p(&self) -> bool { + self._90p + } + + pub fn _75p(&self) -> bool { + self._75p + } + + pub fn median(&self) -> bool { + self.median + } + + pub fn _25p(&self) -> bool { + self._25p + } + + pub fn _10p(&self) -> bool { + self._10p + } + + pub fn min(&self) -> bool { + self.min + } + + pub fn first(&self) -> bool { + self.first + } + + pub fn last(&self) -> bool { + self.last + } + + pub fn cumulative(&self) -> bool { + self.cumulative + } + pub fn add_first(mut self) -> Self { self.first = true; self diff --git a/crates/brk_computer/src/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/vecs/grouped/from_dateindex.rs index a99a1e066..77a366e60 100644 --- a/crates/brk_computer/src/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/vecs/grouped/from_dateindex.rs @@ -1,15 +1,18 @@ use std::path::Path; use brk_core::{ - DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, Version, WeekIndex, YearIndex, + DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex, + YearIndex, }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format}; +use brk_vec::{ + AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, +}; -use crate::vecs::{Indexes, indexes}; +use crate::vecs::{Indexes, grouped::ComputedVecBuilder, indexes}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedVecsFromDateIndex @@ -17,33 +20,37 @@ where T: ComputedType + PartialOrd, { pub dateindex: Option>, - pub dateindex_extra: ComputedVecBuilder, - pub weekindex: ComputedVecBuilder, - pub monthindex: ComputedVecBuilder, - pub quarterindex: ComputedVecBuilder, - pub yearindex: ComputedVecBuilder, - pub decadeindex: ComputedVecBuilder, + pub dateindex_extra: EagerVecBuilder, + pub weekindex: ComputedVecBuilder, + pub monthindex: ComputedVecBuilder, + pub quarterindex: ComputedVecBuilder, + pub semesterindex: ComputedVecBuilder, + pub yearindex: ComputedVecBuilder, + pub decadeindex: ComputedVecBuilder, } const VERSION: Version = Version::ZERO; impl ComputedVecsFromDateIndex where - T: ComputedType, + T: ComputedType + 'static, { + #[allow(clippy::too_many_arguments)] pub fn forced_import( path: &Path, name: &str, - compute_source: bool, + source: Source, version: Version, format: Format, - options: StorableVecGeneatorOptions, + computation: Computation, + indexes: &indexes::Vecs, + options: VecBuilderOptions, ) -> color_eyre::Result { - let dateindex = compute_source.then(|| { + let dateindex = source.is_compute().then(|| { EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap() }); - let dateindex_extra = ComputedVecBuilder::forced_import( + let dateindex_extra = EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, @@ -53,44 +60,77 @@ where let options = options.remove_percentiles(); + let dateindex_source = source.vec().or(dateindex.as_ref().map(|v| v.boxed_clone())); + Ok(Self { - dateindex, - dateindex_extra, weekindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + computation, + dateindex_source.clone(), + &dateindex_extra, + indexes.weekindex_to_weekindex.boxed_clone(), + options.into(), )?, monthindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + dateindex_source.clone(), + &dateindex_extra, + indexes.monthindex_to_monthindex.boxed_clone(), + options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + dateindex_source.clone(), + &dateindex_extra, + indexes.quarterindex_to_quarterindex.boxed_clone(), + options.into(), + )?, + semesterindex: ComputedVecBuilder::forced_import( + path, + name, + version + VERSION + Version::ZERO, + format, + Computation::Lazy, + dateindex_source.clone(), + &dateindex_extra, + indexes.semesterindex_to_semesterindex.boxed_clone(), + options.into(), )?, yearindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + dateindex_source.clone(), + &dateindex_extra, + indexes.yearindex_to_yearindex.boxed_clone(), + options.into(), )?, decadeindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + dateindex_source.clone(), + &dateindex_extra, + indexes.decadeindex_to_decadeindex.boxed_clone(), + options.into(), )?, + dateindex, + dateindex_extra, }) } @@ -133,65 +173,45 @@ where if let Some(dateindex) = dateindex { self.dateindex_extra .extend(starting_indexes.dateindex, dateindex, exit)?; - - self.weekindex.compute( - starting_indexes.weekindex, - dateindex, - &indexes.weekindex_to_first_dateindex, - &indexes.weekindex_to_dateindex_count, - exit, - )?; - - self.monthindex.compute( - starting_indexes.monthindex, - dateindex, - &indexes.monthindex_to_first_dateindex, - &indexes.monthindex_to_dateindex_count, - exit, - )?; } else { let dateindex = self.dateindex.as_ref().unwrap(); self.dateindex_extra .extend(starting_indexes.dateindex, dateindex, exit)?; - - self.weekindex.compute( - starting_indexes.weekindex, - dateindex, - &indexes.weekindex_to_first_dateindex, - &indexes.weekindex_to_dateindex_count, - exit, - )?; - - self.monthindex.compute( - starting_indexes.monthindex, - dateindex, - &indexes.monthindex_to_first_dateindex, - &indexes.monthindex_to_dateindex_count, - exit, - )?; } - self.quarterindex.from_aligned( + self.weekindex.compute_if_necessary( + starting_indexes.weekindex, + &indexes.weekindex_to_dateindex_count, + exit, + )?; + + self.monthindex.compute_if_necessary( + starting_indexes.monthindex, + &indexes.monthindex_to_dateindex_count, + exit, + )?; + + self.quarterindex.compute_if_necessary( starting_indexes.quarterindex, - &self.monthindex, - &indexes.quarterindex_to_first_monthindex, &indexes.quarterindex_to_monthindex_count, exit, )?; - self.yearindex.from_aligned( + self.semesterindex.compute_if_necessary( + starting_indexes.semesterindex, + &indexes.semesterindex_to_monthindex_count, + exit, + )?; + + self.yearindex.compute_if_necessary( starting_indexes.yearindex, - &self.monthindex, - &indexes.yearindex_to_first_monthindex, &indexes.yearindex_to_monthindex_count, exit, )?; - self.decadeindex.from_aligned( + self.decadeindex.compute_if_necessary( starting_indexes.decadeindex, - &self.yearindex, - &indexes.decadeindex_to_first_yearindex, &indexes.decadeindex_to_yearindex_count, exit, )?; @@ -208,6 +228,7 @@ where self.weekindex.vecs(), self.monthindex.vecs(), self.quarterindex.vecs(), + self.semesterindex.vecs(), self.yearindex.vecs(), self.decadeindex.vecs(), ] diff --git a/crates/brk_computer/src/vecs/grouped/from_height.rs b/crates/brk_computer/src/vecs/grouped/from_height.rs index a8ed7e0d7..0ed075529 100644 --- a/crates/brk_computer/src/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/vecs/grouped/from_height.rs @@ -1,16 +1,22 @@ use std::path::Path; use brk_core::{ - DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, Version, - WeekIndex, YearIndex, + DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, + SemesterIndex, Version, WeekIndex, YearIndex, }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format}; +use brk_vec::{ + AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, +}; -use crate::vecs::{Indexes, indexes}; +use crate::vecs::{ + Indexes, + grouped::{ComputedVecBuilder, Source}, + indexes, +}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, EagerVecBuilder, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedVecsFromHeight @@ -18,37 +24,41 @@ where T: ComputedType + PartialOrd, { pub height: Option>, - pub height_extra: ComputedVecBuilder, - pub dateindex: ComputedVecBuilder, - pub weekindex: ComputedVecBuilder, - pub difficultyepoch: ComputedVecBuilder, - pub monthindex: ComputedVecBuilder, - pub quarterindex: ComputedVecBuilder, - pub yearindex: ComputedVecBuilder, + pub height_extra: EagerVecBuilder, + pub dateindex: EagerVecBuilder, + pub weekindex: ComputedVecBuilder, + pub difficultyepoch: EagerVecBuilder, + pub monthindex: ComputedVecBuilder, + pub quarterindex: ComputedVecBuilder, + pub semesterindex: ComputedVecBuilder, + pub yearindex: ComputedVecBuilder, // TODO: pub halvingepoch: StorableVecGeneator, - pub decadeindex: ComputedVecBuilder, + pub decadeindex: ComputedVecBuilder, } const VERSION: Version = Version::ZERO; impl ComputedVecsFromHeight where - T: ComputedType + Ord + From, + T: ComputedType + Ord + From + 'static, f64: From, { + #[allow(clippy::too_many_arguments)] pub fn forced_import( path: &Path, name: &str, - compute_source: bool, + source: Source, version: Version, format: Format, - options: StorableVecGeneatorOptions, + computation: Computation, + indexes: &indexes::Vecs, + options: VecBuilderOptions, ) -> color_eyre::Result { - let height = compute_source.then(|| { + let height = source.is_compute().then(|| { EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap() }); - let height_extra = ComputedVecBuilder::forced_import( + let height_extra = EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, @@ -56,7 +66,7 @@ where options.copy_self_extra(), )?; - let dateindex = ComputedVecBuilder::forced_import( + let dateindex = EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, @@ -67,46 +77,77 @@ where let options = options.remove_percentiles(); Ok(Self { - height, - height_extra, - dateindex, weekindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, - )?, - difficultyepoch: ComputedVecBuilder::forced_import( - path, - name, - version + VERSION + Version::ZERO, - format, - options, + computation, + None, + &dateindex, + indexes.weekindex_to_weekindex.boxed_clone(), + options.into(), )?, monthindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + None, + &dateindex, + indexes.monthindex_to_monthindex.boxed_clone(), + options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + None, + &dateindex, + indexes.quarterindex_to_quarterindex.boxed_clone(), + options.into(), + )?, + semesterindex: ComputedVecBuilder::forced_import( + path, + name, + version + VERSION + Version::ZERO, + format, + Computation::Lazy, + None, + &dateindex, + indexes.semesterindex_to_semesterindex.boxed_clone(), + options.into(), )?, yearindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + None, + &dateindex, + indexes.yearindex_to_yearindex.boxed_clone(), + options.into(), + )?, + decadeindex: ComputedVecBuilder::forced_import( + path, + name, + version + VERSION + Version::ZERO, + format, + Computation::Lazy, + None, + &dateindex, + indexes.decadeindex_to_decadeindex.boxed_clone(), + options.into(), )?, // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, - decadeindex: ComputedVecBuilder::forced_import( + height, + height_extra, + dateindex, + difficultyepoch: EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, @@ -144,9 +185,9 @@ where indexes: &indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, - height: Option<&impl AnyIterableVec>, + height_vec: Option<&impl AnyIterableVec>, ) -> color_eyre::Result<()> { - if let Some(height) = height { + if let Some(height) = height_vec { self.height_extra .extend(starting_indexes.height, height, exit)?; @@ -188,42 +229,38 @@ where )?; } - self.weekindex.from_aligned( + self.weekindex.compute_if_necessary( starting_indexes.weekindex, - &self.dateindex, - &indexes.weekindex_to_first_dateindex, &indexes.weekindex_to_dateindex_count, exit, )?; - self.monthindex.from_aligned( + self.monthindex.compute_if_necessary( starting_indexes.monthindex, - &self.dateindex, - &indexes.monthindex_to_first_dateindex, &indexes.monthindex_to_dateindex_count, exit, )?; - self.quarterindex.from_aligned( + self.quarterindex.compute_if_necessary( starting_indexes.quarterindex, - &self.monthindex, - &indexes.quarterindex_to_first_monthindex, &indexes.quarterindex_to_monthindex_count, exit, )?; - self.yearindex.from_aligned( + self.semesterindex.compute_if_necessary( + starting_indexes.semesterindex, + &indexes.semesterindex_to_monthindex_count, + exit, + )?; + + self.yearindex.compute_if_necessary( starting_indexes.yearindex, - &self.monthindex, - &indexes.yearindex_to_first_monthindex, &indexes.yearindex_to_monthindex_count, exit, )?; - self.decadeindex.from_aligned( + self.decadeindex.compute_if_necessary( starting_indexes.decadeindex, - &self.yearindex, - &indexes.decadeindex_to_first_yearindex, &indexes.decadeindex_to_yearindex_count, exit, )?; @@ -242,6 +279,7 @@ where self.difficultyepoch.vecs(), self.monthindex.vecs(), self.quarterindex.vecs(), + self.semesterindex.vecs(), self.yearindex.vecs(), // self.halvingepoch.vecs(), self.decadeindex.vecs(), diff --git a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/vecs/grouped/from_height_strict.rs index 845a2d08e..1147a7f54 100644 --- a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/vecs/grouped/from_height_strict.rs @@ -7,7 +7,7 @@ use brk_vec::{AnyCollectableVec, EagerVec, Format}; use crate::vecs::{Indexes, indexes}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, EagerVecBuilder, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedVecsFromHeightStrict @@ -15,8 +15,8 @@ where T: ComputedType + PartialOrd, { pub height: EagerVec, - pub height_extra: ComputedVecBuilder, - pub difficultyepoch: ComputedVecBuilder, + pub height_extra: EagerVecBuilder, + pub difficultyepoch: EagerVecBuilder, // TODO: pub halvingepoch: StorableVecGeneator, } @@ -32,12 +32,12 @@ where name: &str, version: Version, format: Format, - options: StorableVecGeneatorOptions, + options: VecBuilderOptions, ) -> color_eyre::Result { let height = EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?; - let height_extra = ComputedVecBuilder::forced_import( + let height_extra = EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, @@ -50,7 +50,7 @@ where Ok(Self { height, height_extra, - difficultyepoch: ComputedVecBuilder::forced_import( + difficultyepoch: EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, diff --git a/crates/brk_computer/src/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/vecs/grouped/from_txindex.rs index 031b95828..a9eb32948 100644 --- a/crates/brk_computer/src/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/vecs/grouped/from_txindex.rs @@ -2,17 +2,22 @@ use std::path::Path; use brk_core::{ Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex, - Result, Sats, TxIndex, Version, WeekIndex, YearIndex, + Result, Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex, }; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{ - AnyCollectableVec, AnyVec, CollectableVec, EagerVec, Format, StoredIndex, VecIterator, + AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec, + Format, StoredIndex, VecIterator, }; -use crate::vecs::{Indexes, fetched, indexes}; +use crate::vecs::{ + Indexes, fetched, + grouped::{ComputedVecBuilder, Source}, + indexes, +}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, EagerVecBuilder, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedVecsFromTxindex @@ -20,40 +25,44 @@ where T: ComputedType + PartialOrd, { pub txindex: Option>>, - pub height: ComputedVecBuilder, - pub dateindex: ComputedVecBuilder, - pub weekindex: ComputedVecBuilder, - pub difficultyepoch: ComputedVecBuilder, - pub monthindex: ComputedVecBuilder, - pub quarterindex: ComputedVecBuilder, - pub yearindex: ComputedVecBuilder, + pub height: EagerVecBuilder, + pub dateindex: EagerVecBuilder, + pub weekindex: ComputedVecBuilder, + pub difficultyepoch: EagerVecBuilder, + pub monthindex: ComputedVecBuilder, + pub quarterindex: ComputedVecBuilder, + pub semesterindex: ComputedVecBuilder, + pub yearindex: ComputedVecBuilder, // TODO: pub halvingepoch: StorableVecGeneator, - pub decadeindex: ComputedVecBuilder, + pub decadeindex: ComputedVecBuilder, } const VERSION: Version = Version::ZERO; impl ComputedVecsFromTxindex where - T: ComputedType + Ord + From, + T: ComputedType + Ord + From + 'static, f64: From, { + #[allow(clippy::too_many_arguments)] pub fn forced_import( path: &Path, name: &str, - compute_source: bool, + source: Source, version: Version, format: Format, - options: StorableVecGeneatorOptions, + computation: Computation, + indexes: &indexes::Vecs, + options: VecBuilderOptions, ) -> color_eyre::Result { - let txindex = compute_source.then(|| { + let txindex = source.is_compute().then(|| { Box::new( EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format) .unwrap(), ) }); - let height = ComputedVecBuilder::forced_import( + let height = EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, @@ -63,45 +72,86 @@ where let options = options.remove_percentiles(); + let dateindex = EagerVecBuilder::forced_import( + path, + name, + version + VERSION + Version::ZERO, + format, + options, + )?; + Ok(Self { - txindex, - height, - dateindex: ComputedVecBuilder::forced_import( - path, - name, - version + VERSION + Version::ZERO, - format, - options, - )?, weekindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, - )?, - difficultyepoch: ComputedVecBuilder::forced_import( - path, - name, - version + VERSION + Version::ZERO, - format, - options, + computation, + None, + &dateindex, + indexes.weekindex_to_weekindex.boxed_clone(), + options.into(), )?, monthindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + None, + &dateindex, + indexes.monthindex_to_monthindex.boxed_clone(), + options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, format, - options, + Computation::Lazy, + None, + &dateindex, + indexes.quarterindex_to_quarterindex.boxed_clone(), + options.into(), + )?, + semesterindex: ComputedVecBuilder::forced_import( + path, + name, + version + VERSION + Version::ZERO, + format, + Computation::Lazy, + None, + &dateindex, + indexes.semesterindex_to_semesterindex.boxed_clone(), + options.into(), )?, yearindex: ComputedVecBuilder::forced_import( + path, + name, + version + VERSION + Version::ZERO, + format, + Computation::Lazy, + None, + &dateindex, + indexes.yearindex_to_yearindex.boxed_clone(), + options.into(), + )?, + decadeindex: ComputedVecBuilder::forced_import( + path, + name, + version + VERSION + Version::ZERO, + format, + Computation::Lazy, + None, + &dateindex, + indexes.decadeindex_to_decadeindex.boxed_clone(), + options.into(), + )?, + + txindex, + height, + dateindex, + difficultyepoch: EagerVecBuilder::forced_import( path, name, version + VERSION + Version::ZERO, @@ -109,13 +159,6 @@ where options, )?, // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, - decadeindex: ComputedVecBuilder::forced_import( - path, - name, - version + VERSION + Version::ZERO, - format, - options, - )?, }) } @@ -196,42 +239,38 @@ where exit, )?; - self.weekindex.from_aligned( + self.weekindex.compute_if_necessary( starting_indexes.weekindex, - &self.dateindex, &indexes.weekindex_to_first_dateindex, - &indexes.weekindex_to_dateindex_count, exit, )?; - self.monthindex.from_aligned( + self.monthindex.compute_if_necessary( starting_indexes.monthindex, - &self.dateindex, - &indexes.monthindex_to_first_dateindex, &indexes.monthindex_to_dateindex_count, exit, )?; - self.quarterindex.from_aligned( + self.quarterindex.compute_if_necessary( starting_indexes.quarterindex, - &self.monthindex, - &indexes.quarterindex_to_first_monthindex, &indexes.quarterindex_to_monthindex_count, exit, )?; - self.yearindex.from_aligned( + self.semesterindex.compute_if_necessary( + starting_indexes.semesterindex, + &indexes.semesterindex_to_monthindex_count, + exit, + )?; + + self.yearindex.compute_if_necessary( starting_indexes.yearindex, - &self.monthindex, - &indexes.yearindex_to_first_monthindex, &indexes.yearindex_to_monthindex_count, exit, )?; - self.decadeindex.from_aligned( + self.decadeindex.compute_if_necessary( starting_indexes.decadeindex, - &self.yearindex, - &indexes.decadeindex_to_first_yearindex, &indexes.decadeindex_to_yearindex_count, exit, )?; @@ -258,6 +297,7 @@ where self.difficultyepoch.vecs(), self.monthindex.vecs(), self.quarterindex.vecs(), + self.semesterindex.vecs(), self.yearindex.vecs(), // self.halvingepoch.vecs(), self.decadeindex.vecs(), diff --git a/crates/brk_computer/src/vecs/grouped/mod.rs b/crates/brk_computer/src/vecs/grouped/mod.rs index 10457ca49..5c4e67968 100644 --- a/crates/brk_computer/src/vecs/grouped/mod.rs +++ b/crates/brk_computer/src/vecs/grouped/mod.rs @@ -1,21 +1,25 @@ -mod builder; +mod builder_computed; +mod builder_eager; mod from_dateindex; mod from_height; mod from_height_strict; mod from_txindex; mod ratio_from_dateindex; +mod source; mod r#type; mod value_from_dateindex; mod value_from_height; mod value_from_txindex; mod value_height; -pub use builder::*; +pub use builder_computed::*; +pub use builder_eager::*; pub use from_dateindex::*; pub use from_height::*; pub use from_height_strict::*; pub use from_txindex::*; pub use ratio_from_dateindex::*; +pub use source::*; use r#type::*; pub use value_from_dateindex::*; pub use value_from_height::*; diff --git a/crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs index 75fa03460..1bc7f6f77 100644 --- a/crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs @@ -4,80 +4,86 @@ use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{ - AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, EagerVec, Format, StoredIndex, - VecIterator, + AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, + StoredIndex, VecIterator, }; use crate::{ utils::get_percentile, - vecs::{Indexes, fetched, indexes}, + vecs::{Indexes, fetched, grouped::source::Source, indexes}, }; -use super::{ComputedVecsFromDateIndex, StorableVecGeneatorOptions}; +use super::{ComputedVecsFromDateIndex, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedRatioVecsFromDateIndex { pub price: Option>, pub ratio: ComputedVecsFromDateIndex, - pub ratio_sma: ComputedVecsFromDateIndex, - pub ratio_1w_sma: ComputedVecsFromDateIndex, - pub ratio_1m_sma: ComputedVecsFromDateIndex, - pub ratio_1y_sma: ComputedVecsFromDateIndex, - pub ratio_4y_sma: ComputedVecsFromDateIndex, - pub ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex, - pub ratio_sd: ComputedVecsFromDateIndex, - pub ratio_4y_sd: ComputedVecsFromDateIndex, - pub ratio_1y_sd: ComputedVecsFromDateIndex, - pub ratio_p99_9: ComputedVecsFromDateIndex, - pub ratio_p99_5: ComputedVecsFromDateIndex, - pub ratio_p99: ComputedVecsFromDateIndex, - pub ratio_p1: ComputedVecsFromDateIndex, - pub ratio_p0_5: ComputedVecsFromDateIndex, - pub ratio_p0_1: ComputedVecsFromDateIndex, - pub ratio_p1sd: ComputedVecsFromDateIndex, - pub ratio_p2sd: ComputedVecsFromDateIndex, - pub ratio_p3sd: ComputedVecsFromDateIndex, - pub ratio_m1sd: ComputedVecsFromDateIndex, - pub ratio_m2sd: ComputedVecsFromDateIndex, - pub ratio_m3sd: ComputedVecsFromDateIndex, - pub ratio_p99_9_as_price: ComputedVecsFromDateIndex, - pub ratio_p99_5_as_price: ComputedVecsFromDateIndex, - pub ratio_p99_as_price: ComputedVecsFromDateIndex, - pub ratio_p1_as_price: ComputedVecsFromDateIndex, - pub ratio_p0_5_as_price: ComputedVecsFromDateIndex, - pub ratio_p0_1_as_price: ComputedVecsFromDateIndex, - pub ratio_p1sd_as_price: ComputedVecsFromDateIndex, - pub ratio_p2sd_as_price: ComputedVecsFromDateIndex, - pub ratio_p3sd_as_price: ComputedVecsFromDateIndex, - pub ratio_m1sd_as_price: ComputedVecsFromDateIndex, - pub ratio_m2sd_as_price: ComputedVecsFromDateIndex, - pub ratio_m3sd_as_price: ComputedVecsFromDateIndex, - pub ratio_zscore: ComputedVecsFromDateIndex, - pub ratio_4y_zscore: ComputedVecsFromDateIndex, - pub ratio_1y_zscore: ComputedVecsFromDateIndex, + pub ratio_sma: Option>, + pub ratio_1w_sma: Option>, + pub ratio_1m_sma: Option>, + pub ratio_1y_sma: Option>, + pub ratio_4y_sma: Option>, + pub ratio_1y_sma_momentum_oscillator: Option>, + pub ratio_sd: Option>, + pub ratio_4y_sd: Option>, + pub ratio_1y_sd: Option>, + pub ratio_p99_9: Option>, + pub ratio_p99_5: Option>, + pub ratio_p99: Option>, + pub ratio_p1: Option>, + pub ratio_p0_5: Option>, + pub ratio_p0_1: Option>, + pub ratio_p1sd: Option>, + pub ratio_p2sd: Option>, + pub ratio_p3sd: Option>, + pub ratio_m1sd: Option>, + pub ratio_m2sd: Option>, + pub ratio_m3sd: Option>, + pub ratio_p99_9_as_price: Option>, + pub ratio_p99_5_as_price: Option>, + pub ratio_p99_as_price: Option>, + pub ratio_p1_as_price: Option>, + pub ratio_p0_5_as_price: Option>, + pub ratio_p0_1_as_price: Option>, + pub ratio_p1sd_as_price: Option>, + pub ratio_p2sd_as_price: Option>, + pub ratio_p3sd_as_price: Option>, + pub ratio_m1sd_as_price: Option>, + pub ratio_m2sd_as_price: Option>, + pub ratio_m3sd_as_price: Option>, + pub ratio_zscore: Option>, + pub ratio_4y_zscore: Option>, + pub ratio_1y_zscore: Option>, } const VERSION: Version = Version::ZERO; impl ComputedRatioVecsFromDateIndex { + #[allow(clippy::too_many_arguments)] pub fn forced_import( path: &Path, name: &str, - compute_source: bool, + source: Source, version: Version, format: Format, + computation: Computation, + indexes: &indexes::Vecs, + extended: bool, ) -> color_eyre::Result { - let options = StorableVecGeneatorOptions::default().add_last(); + let options = VecBuilderOptions::default().add_last(); Ok(Self { - price: compute_source.then(|| { + price: source.is_compute().then(|| { ComputedVecsFromDateIndex::forced_import( path, name, - true, + Source::Compute, version + VERSION, format, + computation, + indexes, options, ) .unwrap() @@ -85,299 +91,481 @@ impl ComputedRatioVecsFromDateIndex { ratio: ComputedVecsFromDateIndex::forced_import( path, &format!("{name}_ratio"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_sma: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_sma"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_1w_sma: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_1w_sma"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_1m_sma: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_1m_sma"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_1y_sma: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_1y_sma"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_4y_sma: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_4y_sma"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_1y_sma_momentum_oscillator"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_4y_sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_4y_sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_1y_sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_1y_sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p99_9: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p99_9"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p99_5: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p99_5"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p99: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p99"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p1: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p1"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p0_5: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p0_5"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p0_1: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p0_1"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p1sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p1sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p2sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p2sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p3sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p3sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_m1sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_m1sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_m2sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_m2sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_m3sd: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_m3sd"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p99_9_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p99_9_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p99_5_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p99_5_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p99_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p99_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p1_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p1_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p0_5_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p0_5_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p0_1_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p0_1_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p1sd_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p1sd_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p2sd_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p2sd_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_p3sd_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_p3sd_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_m1sd_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_m1sd_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_m2sd_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_m2sd_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_m3sd_as_price: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_m3sd_as_price"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_zscore: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_zscore"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_4y_zscore: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_4y_zscore"), - true, - version + VERSION + Version::ZERO, - format, - options, - )?, - ratio_1y_zscore: ComputedVecsFromDateIndex::forced_import( - path, - &format!("{name}_ratio_1y_zscore"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, options, )?, + ratio_sma: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_sma"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_1w_sma: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_1w_sma"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_1m_sma: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_1m_sma"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_1y_sma: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_1y_sma"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_4y_sma: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_4y_sma"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_1y_sma_momentum_oscillator: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_1y_sma_momentum_oscillator"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_4y_sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_4y_sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_1y_sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_1y_sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p99_9: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p99_9"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p99_5: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p99_5"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p99: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p99"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p1: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p1"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p0_5: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p0_5"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p0_1: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p0_1"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p1sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p1sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p2sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p2sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p3sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p3sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_m1sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_m1sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_m2sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_m2sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_m3sd: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_m3sd"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p99_9_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p99_9_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p99_5_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p99_5_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p99_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p99_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p1_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p1_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p0_5_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p0_5_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p0_1_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p0_1_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p1sd_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p1sd_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p2sd_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p2sd_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_p3sd_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_p3sd_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_m1sd_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_m1sd_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_m2sd_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_m2sd_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_m3sd_as_price: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_m3sd_as_price"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_zscore: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_zscore"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_4y_zscore: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_4y_zscore"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), + ratio_1y_zscore: extended.then(|| { + ComputedVecsFromDateIndex::forced_import( + path, + &format!("{name}_ratio_1y_zscore"), + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + options, + ) + .unwrap() + }), }) } @@ -456,9 +644,13 @@ impl ComputedRatioVecsFromDateIndex { }, )?; + if self.ratio_sma.is_none() { + return Ok(()); + } + let min_ratio_date = DateIndex::try_from(Date::MIN_RATIO).unwrap(); - self.ratio_sma.compute_all( + self.ratio_sma.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -474,7 +666,7 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_1w_sma.compute_all( + self.ratio_1w_sma.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -490,7 +682,7 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_1m_sma.compute_all( + self.ratio_1m_sma.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -506,7 +698,7 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_1y_sma.compute_all( + self.ratio_1y_sma.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -522,7 +714,7 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_4y_sma.compute_all( + self.ratio_4y_sma.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -538,27 +730,38 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_1y_sma_momentum_oscillator.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |v, _, _, starting_indexes, exit| { - let mut ratio_1y_sma_iter = - self.ratio_1y_sma.dateindex.as_ref().unwrap().into_iter(); - v.compute_transform( - starting_indexes.dateindex, - self.ratio.dateindex.as_ref().unwrap(), - |(i, ratio, ..)| { - ( - i, - StoredF32::from(*ratio / *ratio_1y_sma_iter.unwrap_get_inner(i) - 1.0), - ) - }, - exit, - ) - }, - )?; + self.ratio_1y_sma_momentum_oscillator + .as_mut() + .unwrap() + .compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + let mut ratio_1y_sma_iter = self + .ratio_1y_sma + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); + v.compute_transform( + starting_indexes.dateindex, + self.ratio.dateindex.as_ref().unwrap(), + |(i, ratio, ..)| { + ( + i, + StoredF32::from( + *ratio / *ratio_1y_sma_iter.unwrap_get_inner(i) - 1.0, + ), + ) + }, + exit, + ) + }, + )?; let ratio_version = self.ratio.dateindex.as_ref().unwrap().version(); self.mut_ratio_vecs() @@ -588,9 +791,30 @@ impl ComputedRatioVecsFromDateIndex { // unreachable!(); // } - let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter(); - let mut _4y_sma_iter = self.ratio_4y_sma.dateindex.as_ref().unwrap().into_iter(); - let mut _1y_sma_iter = self.ratio_1y_sma.dateindex.as_ref().unwrap().into_iter(); + let mut sma_iter = self + .ratio_sma + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); + let mut _4y_sma_iter = self + .ratio_4y_sma + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); + let mut _1y_sma_iter = self + .ratio_1y_sma + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); let nan = StoredF32::from(f32::NAN); self.ratio @@ -601,111 +825,153 @@ impl ComputedRatioVecsFromDateIndex { .try_for_each(|(index, ratio)| -> Result<()> { if index < min_ratio_date { self.ratio_p0_1 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p0_5 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p1 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p99 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p99_5 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p99_9 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_4y_sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_1y_sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p1sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p2sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_p3sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_m1sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_m2sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; self.ratio_m3sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, nan, exit)?; } else { - let ratio = ratio.into_inner(); + let ratio = ratio.into_owned(); let pos = sorted.binary_search(&ratio).unwrap_or_else(|pos| pos); sorted.insert(pos, ratio); - self.ratio_p0_1.dateindex.as_mut().unwrap().forced_push_at( - index, - get_percentile(&sorted, 0.001), - exit, - )?; - self.ratio_p0_5.dateindex.as_mut().unwrap().forced_push_at( - index, - get_percentile(&sorted, 0.005), - exit, - )?; - self.ratio_p1.dateindex.as_mut().unwrap().forced_push_at( - index, - get_percentile(&sorted, 0.01), - exit, - )?; - self.ratio_p99.dateindex.as_mut().unwrap().forced_push_at( - index, - get_percentile(&sorted, 0.99), - exit, - )?; + self.ratio_p0_1 + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, get_percentile(&sorted, 0.001), exit)?; + self.ratio_p0_5 + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, get_percentile(&sorted, 0.005), exit)?; + self.ratio_p1 + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, get_percentile(&sorted, 0.01), exit)?; + self.ratio_p99 + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, get_percentile(&sorted, 0.99), exit)?; self.ratio_p99_5 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, get_percentile(&sorted, 0.995), exit)?; self.ratio_p99_9 + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() @@ -720,6 +986,8 @@ impl ComputedRatioVecsFromDateIndex { ); self.ratio_sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() @@ -734,6 +1002,8 @@ impl ComputedRatioVecsFromDateIndex { ); self.ratio_4y_sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() @@ -748,41 +1018,55 @@ impl ComputedRatioVecsFromDateIndex { ); self.ratio_1y_sd + .as_mut() + .unwrap() .dateindex .as_mut() .unwrap() .forced_push_at(index, _1y_sd, exit)?; - self.ratio_p1sd.dateindex.as_mut().unwrap().forced_push_at( - index, - avg + sd, - exit, - )?; - self.ratio_p2sd.dateindex.as_mut().unwrap().forced_push_at( - index, - avg + 2 * sd, - exit, - )?; - self.ratio_p3sd.dateindex.as_mut().unwrap().forced_push_at( - index, - avg + 3 * sd, - exit, - )?; - self.ratio_m1sd.dateindex.as_mut().unwrap().forced_push_at( - index, - avg - sd, - exit, - )?; - self.ratio_m2sd.dateindex.as_mut().unwrap().forced_push_at( - index, - avg - 2 * sd, - exit, - )?; - self.ratio_m3sd.dateindex.as_mut().unwrap().forced_push_at( - index, - avg - 3 * sd, - exit, - )?; + self.ratio_p1sd + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, avg + sd, exit)?; + self.ratio_p2sd + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, avg + 2 * sd, exit)?; + self.ratio_p3sd + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, avg + 3 * sd, exit)?; + self.ratio_m1sd + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, avg - sd, exit)?; + self.ratio_m2sd + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, avg - 2 * sd, exit)?; + self.ratio_m3sd + .as_mut() + .unwrap() + .dateindex + .as_mut() + .unwrap() + .forced_push_at(index, avg - 3 * sd, exit)?; } Ok(()) @@ -792,91 +1076,91 @@ impl ComputedRatioVecsFromDateIndex { .into_iter() .try_for_each(|v| v.safe_flush(exit))?; - self.ratio_p99_9.compute_rest( + self.ratio_p99_9.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p99_5.compute_rest( + self.ratio_p99_5.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p99.compute_rest( + self.ratio_p99.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p1.compute_rest( + self.ratio_p1.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p0_5.compute_rest( + self.ratio_p0_5.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p0_1.compute_rest( + self.ratio_p0_1.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_sd.compute_rest( + self.ratio_sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_4y_sd.compute_rest( + self.ratio_4y_sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_1y_sd.compute_rest( + self.ratio_1y_sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p1sd.compute_rest( + self.ratio_p1sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p2sd.compute_rest( + self.ratio_p2sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_p3sd.compute_rest( + self.ratio_p3sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_m1sd.compute_rest( + self.ratio_m1sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_m2sd.compute_rest( + self.ratio_m2sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, None as Option<&EagerVec<_, _>>, )?; - self.ratio_m3sd.compute_rest( + self.ratio_m3sd.as_mut().unwrap().compute_rest( indexes, starting_indexes, exit, @@ -887,13 +1171,20 @@ impl ComputedRatioVecsFromDateIndex { std::mem::transmute(&self.price.as_ref().unwrap().dateindex) }); - self.ratio_p99_as_price.compute_all( + self.ratio_p99_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p99.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p99 + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -906,13 +1197,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p99_5_as_price.compute_all( + self.ratio_p99_5_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p99_5.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p99_5 + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -925,13 +1223,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p99_9_as_price.compute_all( + self.ratio_p99_9_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p99_9.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p99_9 + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -944,13 +1249,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p1_as_price.compute_all( + self.ratio_p1_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p1.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p1 + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -963,13 +1275,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p0_5_as_price.compute_all( + self.ratio_p0_5_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p0_5.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p0_5 + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -982,13 +1301,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p0_1_as_price.compute_all( + self.ratio_p0_1_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p0_1.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p0_1 + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -1001,13 +1327,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p1sd_as_price.compute_all( + self.ratio_p1sd_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p1sd.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p1sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -1020,13 +1353,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p2sd_as_price.compute_all( + self.ratio_p2sd_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p2sd.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p2sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -1039,13 +1379,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_p3sd_as_price.compute_all( + self.ratio_p3sd_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_p3sd.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_p3sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -1058,13 +1405,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_m1sd_as_price.compute_all( + self.ratio_m1sd_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_m1sd.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_m1sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -1077,13 +1431,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_m2sd_as_price.compute_all( + self.ratio_m2sd_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_m2sd.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_m2sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -1096,13 +1457,20 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_m3sd_as_price.compute_all( + self.ratio_m3sd_as_price.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { - let mut iter = self.ratio_m3sd.dateindex.as_ref().unwrap().into_iter(); + let mut iter = self + .ratio_m3sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap() + .into_iter(); vec.compute_transform( starting_indexes.dateindex, date_to_price, @@ -1115,7 +1483,7 @@ impl ComputedRatioVecsFromDateIndex { }, )?; - self.ratio_zscore.compute_all( + self.ratio_zscore.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -1124,14 +1492,14 @@ impl ComputedRatioVecsFromDateIndex { vec.compute_zscore( starting_indexes.dateindex, self.ratio.dateindex.as_ref().unwrap(), - self.ratio_sma.dateindex.as_ref().unwrap(), - self.ratio_sd.dateindex.as_ref().unwrap(), + self.ratio_sma.as_ref().unwrap().dateindex.as_ref().unwrap(), + self.ratio_sd.as_ref().unwrap().dateindex.as_ref().unwrap(), exit, ) }, )?; - self.ratio_4y_zscore.compute_all( + self.ratio_4y_zscore.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -1140,14 +1508,24 @@ impl ComputedRatioVecsFromDateIndex { vec.compute_zscore( starting_indexes.dateindex, self.ratio.dateindex.as_ref().unwrap(), - self.ratio_4y_sma.dateindex.as_ref().unwrap(), - self.ratio_4y_sd.dateindex.as_ref().unwrap(), + self.ratio_4y_sma + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + self.ratio_4y_sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), exit, ) }, )?; - self.ratio_1y_zscore.compute_all( + self.ratio_1y_zscore.as_mut().unwrap().compute_all( indexer, indexes, starting_indexes, @@ -1156,8 +1534,18 @@ impl ComputedRatioVecsFromDateIndex { vec.compute_zscore( starting_indexes.dateindex, self.ratio.dateindex.as_ref().unwrap(), - self.ratio_1y_sma.dateindex.as_ref().unwrap(), - self.ratio_1y_sd.dateindex.as_ref().unwrap(), + self.ratio_1y_sma + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + self.ratio_1y_sd + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), exit, ) }, @@ -1167,65 +1555,122 @@ impl ComputedRatioVecsFromDateIndex { } fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec> { - vec![ - self.ratio_sd.dateindex.as_mut().unwrap(), - self.ratio_4y_sd.dateindex.as_mut().unwrap(), - self.ratio_1y_sd.dateindex.as_mut().unwrap(), - self.ratio_p99_9.dateindex.as_mut().unwrap(), - self.ratio_p99_5.dateindex.as_mut().unwrap(), - self.ratio_p99.dateindex.as_mut().unwrap(), - self.ratio_p1.dateindex.as_mut().unwrap(), - self.ratio_p0_5.dateindex.as_mut().unwrap(), - self.ratio_p0_1.dateindex.as_mut().unwrap(), - self.ratio_p1sd.dateindex.as_mut().unwrap(), - self.ratio_p2sd.dateindex.as_mut().unwrap(), - self.ratio_p3sd.dateindex.as_mut().unwrap(), - self.ratio_m1sd.dateindex.as_mut().unwrap(), - self.ratio_m2sd.dateindex.as_mut().unwrap(), - self.ratio_m3sd.dateindex.as_mut().unwrap(), + [ + self.ratio_sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_4y_sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_1y_sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p99_9 + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p99_5 + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p99 + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p1 + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p0_5 + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p0_1 + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p1sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p2sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_p3sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_m1sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_m2sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), + self.ratio_m3sd + .as_mut() + .map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]), ] + .into_iter() + .flatten() + .collect::>() } pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ self.price.as_ref().map_or(vec![], |v| v.vecs()), self.ratio.vecs(), - self.ratio_sma.vecs(), - self.ratio_1w_sma.vecs(), - self.ratio_1m_sma.vecs(), - self.ratio_1y_sma.vecs(), - self.ratio_4y_sma.vecs(), - self.ratio_1y_sma_momentum_oscillator.vecs(), - self.ratio_sd.vecs(), - self.ratio_1y_sd.vecs(), - self.ratio_4y_sd.vecs(), - self.ratio_p99_9.vecs(), - self.ratio_p99_5.vecs(), - self.ratio_p99.vecs(), - self.ratio_p1.vecs(), - self.ratio_p0_5.vecs(), - self.ratio_p0_1.vecs(), - self.ratio_p1sd.vecs(), - self.ratio_p2sd.vecs(), - self.ratio_p3sd.vecs(), - self.ratio_m1sd.vecs(), - self.ratio_m2sd.vecs(), - self.ratio_m3sd.vecs(), - self.ratio_p99_9_as_price.vecs(), - self.ratio_p99_5_as_price.vecs(), - self.ratio_p99_as_price.vecs(), - self.ratio_p1_as_price.vecs(), - self.ratio_p0_5_as_price.vecs(), - self.ratio_p0_1_as_price.vecs(), - self.ratio_p1sd_as_price.vecs(), - self.ratio_p2sd_as_price.vecs(), - self.ratio_p3sd_as_price.vecs(), - self.ratio_m1sd_as_price.vecs(), - self.ratio_m2sd_as_price.vecs(), - self.ratio_m3sd_as_price.vecs(), - self.ratio_zscore.vecs(), - self.ratio_1y_zscore.vecs(), - self.ratio_4y_zscore.vecs(), + self.ratio_sma.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_1w_sma.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_1m_sma.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_1y_sma.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_4y_sma.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_1y_sma_momentum_oscillator + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_1y_sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_4y_sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p99_9.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p99_5.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p99.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p1.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p0_5.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p0_1.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p1sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p2sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p3sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_m1sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_m2sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_m3sd.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p99_9_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_p99_5_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_p99_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_p1_as_price.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_p0_5_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_p0_1_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_p1sd_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_p2sd_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_p3sd_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_m1sd_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_m2sd_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_m3sd_as_price + .as_ref() + .map_or(vec![], |v| v.vecs()), + self.ratio_zscore.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_1y_zscore.as_ref().map_or(vec![], |v| v.vecs()), + self.ratio_4y_zscore.as_ref().map_or(vec![], |v| v.vecs()), ] .into_iter() .flatten() diff --git a/crates/brk_computer/src/vecs/grouped/source.rs b/crates/brk_computer/src/vecs/grouped/source.rs new file mode 100644 index 000000000..d8886e4f5 --- /dev/null +++ b/crates/brk_computer/src/vecs/grouped/source.rs @@ -0,0 +1,51 @@ +use brk_vec::BoxedAnyIterableVec; + +#[derive(Clone)] +pub enum Source { + Compute, + None, + Vec(BoxedAnyIterableVec), +} + +impl Source { + pub fn is_compute(&self) -> bool { + matches!(self, Self::Compute) + } + + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } + + pub fn is_vec(&self) -> bool { + matches!(self, Self::Vec(_)) + } + + pub fn vec(self) -> Option> { + match self { + Self::Vec(v) => Some(v), + _ => None, + } + } +} + +impl From for Source { + fn from(value: bool) -> Self { + if value { Self::Compute } else { Self::None } + } +} + +impl From> for Source { + fn from(value: BoxedAnyIterableVec) -> Self { + Self::Vec(value) + } +} + +impl From>> for Source { + fn from(value: Option>) -> Self { + if let Some(v) = value { + Self::Vec(v) + } else { + Self::None + } + } +} diff --git a/crates/brk_computer/src/vecs/grouped/type.rs b/crates/brk_computer/src/vecs/grouped/type.rs index c98b6498f..357e6d6ba 100644 --- a/crates/brk_computer/src/vecs/grouped/type.rs +++ b/crates/brk_computer/src/vecs/grouped/type.rs @@ -1,13 +1,14 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, AddAssign, Div}; use brk_vec::StoredType; pub trait ComputedType where - Self: StoredType + From + Div + Add + Ord, + Self: + StoredType + From + Div + Add + AddAssign + Ord, { } impl ComputedType for T where - T: StoredType + From + Div + Add + Ord + T: StoredType + From + Div + Add + AddAssign + Ord { } diff --git a/crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs b/crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs index 87195f2f1..124fec3ea 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs +++ b/crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs @@ -3,11 +3,11 @@ use std::path::Path; use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec}; +use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes}; -use super::StorableVecGeneatorOptions; +use super::{Source, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedValueVecsFromDateIndex { @@ -19,39 +19,48 @@ pub struct ComputedValueVecsFromDateIndex { const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromDateIndex { + #[allow(clippy::too_many_arguments)] pub fn forced_import( path: &Path, name: &str, - compute_source: bool, + source: Source, version: Version, format: Format, - options: StorableVecGeneatorOptions, + computation: Computation, + options: VecBuilderOptions, compute_dollars: bool, + indexes: &indexes::Vecs, ) -> color_eyre::Result { Ok(Self { sats: ComputedVecsFromDateIndex::forced_import( path, name, - compute_source, + source, version + VERSION, format, + computation, + indexes, options, )?, bitcoin: ComputedVecsFromDateIndex::forced_import( path, &format!("{name}_in_btc"), - true, + Source::Compute, version + VERSION, format, + computation, + indexes, options, )?, dollars: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( path, &format!("{name}_in_usd"), - true, + Source::Compute, version + VERSION, format, + computation, + indexes, options, ) .unwrap() diff --git a/crates/brk_computer/src/vecs/grouped/value_from_height.rs b/crates/brk_computer/src/vecs/grouped/value_from_height.rs index 277dafaeb..b582475aa 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_height.rs +++ b/crates/brk_computer/src/vecs/grouped/value_from_height.rs @@ -3,11 +3,11 @@ use std::path::Path; use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec}; +use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; -use crate::vecs::{Indexes, fetched, indexes}; +use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; -use super::{ComputedVecsFromHeight, StorableVecGeneatorOptions}; +use super::{ComputedVecsFromHeight, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedValueVecsFromHeight { @@ -19,39 +19,48 @@ pub struct ComputedValueVecsFromHeight { const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromHeight { + #[allow(clippy::too_many_arguments)] pub fn forced_import( path: &Path, name: &str, - compute_source: bool, + source: Source, version: Version, format: Format, - options: StorableVecGeneatorOptions, + computation: Computation, + options: VecBuilderOptions, compute_dollars: bool, + indexes: &indexes::Vecs, ) -> color_eyre::Result { Ok(Self { sats: ComputedVecsFromHeight::forced_import( path, name, - compute_source, + source, version + VERSION, format, + computation, + indexes, options, )?, bitcoin: ComputedVecsFromHeight::forced_import( path, &format!("{name}_in_btc"), - true, + Source::Compute, version + VERSION, format, + computation, + indexes, options, )?, dollars: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( path, &format!("{name}_in_usd"), - true, + Source::Compute, version + VERSION, format, + computation, + indexes, options, ) .unwrap() diff --git a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs index 91891f98d..534ee33b5 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs @@ -4,13 +4,13 @@ use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{ - AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Computation, - ComputedVecFrom3, Format, LazyVecFrom1, StoredIndex, StoredVec, + AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3, + Format, LazyVecFrom1, StoredIndex, StoredVec, }; -use crate::vecs::{Indexes, fetched, indexes}; +use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; -use super::{ComputedVecsFromTxindex, StorableVecGeneatorOptions}; +use super::{ComputedVecsFromTxindex, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedValueVecsFromTxindex { @@ -41,14 +41,13 @@ impl ComputedValueVecsFromTxindex { path: &Path, name: &str, indexes: &indexes::Vecs, - source: Option>, + source: Source, version: Version, computation: Computation, format: Format, fetched: Option<&fetched::Vecs>, - options: StorableVecGeneatorOptions, + options: VecBuilderOptions, ) -> color_eyre::Result { - let compute_source = source.is_none(); let compute_dollars = fetched.is_some(); let name_in_btc = format!("{name}_in_btc"); @@ -57,19 +56,23 @@ impl ComputedValueVecsFromTxindex { let sats = ComputedVecsFromTxindex::forced_import( path, name, - compute_source, + source.clone(), version + VERSION, format, + computation, + indexes, options, )?; + let source_vec = source.vec(); + let bitcoin_txindex = LazyVecFrom1::init( &name_in_btc, version + VERSION, - source.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s), + source_vec.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s), |txindex: TxIndex, iter| { iter.next_at(txindex.unwrap_to_usize()).map(|(_, value)| { - let sats = value.into_inner(); + let sats = value.into_owned(); Bitcoin::from(sats) }) }, @@ -78,9 +81,11 @@ impl ComputedValueVecsFromTxindex { let bitcoin = ComputedVecsFromTxindex::forced_import( path, &name_in_btc, - false, + Source::None, version + VERSION, format, + computation, + indexes, options, )?; @@ -100,14 +105,14 @@ impl ComputedValueVecsFromTxindex { height_to_close_iter| { let txindex = txindex.unwrap_to_usize(); txindex_to_btc_iter.next_at(txindex).and_then(|(_, value)| { - let btc = value.into_inner(); + let btc = value.into_owned(); txindex_to_height_iter .next_at(txindex) .and_then(|(_, value)| { - let height = value.into_inner(); + let height = value.into_owned(); height_to_close_iter .next_at(height.unwrap_to_usize()) - .map(|(_, close)| *close.into_inner() * btc) + .map(|(_, close)| *close.into_owned() * btc) }) }) }, @@ -124,9 +129,11 @@ impl ComputedValueVecsFromTxindex { ComputedVecsFromTxindex::forced_import( path, &name_in_usd, - false, + Source::None, version + VERSION, format, + computation, + indexes, options, ) .unwrap() diff --git a/crates/brk_computer/src/vecs/grouped/value_height.rs b/crates/brk_computer/src/vecs/grouped/value_height.rs index b5ed7eaaa..bd2c34036 100644 --- a/crates/brk_computer/src/vecs/grouped/value_height.rs +++ b/crates/brk_computer/src/vecs/grouped/value_height.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec}; -use crate::vecs::{Indexes, fetched, indexes}; +use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; #[derive(Clone)] pub struct ComputedHeightValueVecs { @@ -20,13 +20,13 @@ impl ComputedHeightValueVecs { pub fn forced_import( path: &Path, name: &str, - compute_source: bool, + source: Source, version: Version, format: Format, compute_dollars: bool, ) -> color_eyre::Result { Ok(Self { - sats: compute_source.then(|| { + sats: source.is_compute().then(|| { EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format) .unwrap() }), diff --git a/crates/brk_computer/src/vecs/indexes.rs b/crates/brk_computer/src/vecs/indexes.rs index 7b860aa7c..9c7ed5cda 100644 --- a/crates/brk_computer/src/vecs/indexes.rs +++ b/crates/brk_computer/src/vecs/indexes.rs @@ -5,8 +5,8 @@ use brk_core::{ InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2AAddressIndex, P2ABytes, P2MSOutputIndex, P2PK33AddressIndex, P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex, P2PKHBytes, P2SHAddressIndex, P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, - P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, StoredUsize, Timestamp, - TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex, + P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredUsize, + Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex, }; use brk_exit::Exit; use brk_indexer::Indexer; @@ -50,6 +50,7 @@ pub struct Vecs { pub monthindex_to_first_dateindex: EagerVec, pub monthindex_to_monthindex: EagerVec, pub monthindex_to_quarterindex: EagerVec, + pub monthindex_to_semesterindex: EagerVec, pub monthindex_to_yearindex: EagerVec, pub opreturnindex_to_opreturnindex: ComputedVecFrom1, @@ -76,6 +77,9 @@ pub struct Vecs { pub quarterindex_to_first_monthindex: EagerVec, pub quarterindex_to_monthindex_count: EagerVec, pub quarterindex_to_quarterindex: EagerVec, + pub semesterindex_to_first_monthindex: EagerVec, + pub semesterindex_to_monthindex_count: EagerVec, + pub semesterindex_to_semesterindex: EagerVec, pub txindex_to_height: EagerVec, pub txindex_to_input_count: ComputedVecFrom2, @@ -144,10 +148,10 @@ impl Vecs { txindex_to_first_inputindex_iter .next_at(txindex) .map(|(_, start)| { - let start = usize::from(start.into_inner()); + let start = usize::from(start.into_owned()); let end = txindex_to_first_inputindex_iter .next_at(txindex + 1) - .map(|(_, v)| usize::from(v.into_inner())) + .map(|(_, v)| usize::from(v.into_owned())) .unwrap_or_else(|| inputindex_to_outputindex_iter.len()); StoredUsize::from((start..end).count()) }) @@ -167,10 +171,10 @@ impl Vecs { txindex_to_first_outputindex_iter .next_at(txindex) .map(|(_, start)| { - let start = usize::from(start.into_inner()); + let start = usize::from(start.into_owned()); let end = txindex_to_first_outputindex_iter .next_at(txindex + 1) - .map(|(_, v)| usize::from(v.into_inner())) + .map(|(_, v)| usize::from(v.into_owned())) .unwrap_or_else(|| outputindex_to_value_iter.len()); StoredUsize::from((start..end).count()) }) @@ -413,6 +417,12 @@ impl Vecs { version + VERSION + Version::ZERO, format, )?, + monthindex_to_semesterindex: EagerVec::forced_import( + path, + "semesterindex", + version + VERSION + Version::ZERO, + format, + )?, monthindex_to_yearindex: EagerVec::forced_import( path, "yearindex", @@ -425,6 +435,12 @@ impl Vecs { version + VERSION + Version::ZERO, format, )?, + semesterindex_to_first_monthindex: EagerVec::forced_import( + path, + "first_monthindex", + version + VERSION + Version::ZERO, + format, + )?, weekindex_to_first_dateindex: EagerVec::forced_import( path, "first_dateindex", @@ -443,6 +459,12 @@ impl Vecs { version + VERSION + Version::ZERO, format, )?, + semesterindex_to_semesterindex: EagerVec::forced_import( + path, + "semesterindex", + version + VERSION + Version::ZERO, + format, + )?, weekindex_to_weekindex: EagerVec::forced_import( path, "weekindex", @@ -521,6 +543,12 @@ impl Vecs { version + VERSION + Version::ZERO, format, )?, + semesterindex_to_monthindex_count: EagerVec::forced_import( + path, + "monthindex_count", + version + VERSION + Version::ZERO, + format, + )?, yearindex_to_monthindex_count: EagerVec::forced_import( path, "monthindex_count", @@ -943,6 +971,45 @@ impl Vecs { exit, )?; + // --- + // SemesterIndex + // --- + + let starting_semesterindex = self + .monthindex_to_semesterindex + .into_iter() + .get_inner(starting_monthindex) + .unwrap_or_default(); + + self.monthindex_to_semesterindex.compute_from_index( + starting_monthindex, + &self.monthindex_to_first_dateindex, + exit, + )?; + + self.semesterindex_to_first_monthindex + .compute_inverse_more_to_less( + starting_monthindex, + &self.monthindex_to_semesterindex, + exit, + )?; + + // let semester_count = self.semesterindex_to_first_monthindex.len(); + + self.semesterindex_to_semesterindex.compute_from_index( + starting_semesterindex, + &self.semesterindex_to_first_monthindex, + exit, + )?; + + self.semesterindex_to_monthindex_count + .compute_count_from_indexes( + starting_semesterindex, + &self.semesterindex_to_first_monthindex, + &self.monthindex_to_monthindex, + exit, + )?; + // --- // YearIndex // --- @@ -1051,6 +1118,7 @@ impl Vecs { weekindex: starting_weekindex, monthindex: starting_monthindex, quarterindex: starting_quarterindex, + semesterindex: starting_semesterindex, yearindex: starting_yearindex, decadeindex: starting_decadeindex, difficultyepoch: starting_difficultyepoch, @@ -1088,6 +1156,7 @@ impl Vecs { &self.monthindex_to_first_dateindex, &self.monthindex_to_monthindex, &self.monthindex_to_quarterindex, + &self.monthindex_to_semesterindex, &self.monthindex_to_yearindex, &self.opreturnindex_to_opreturnindex, &self.outputindex_to_outputindex, @@ -1103,6 +1172,9 @@ impl Vecs { &self.quarterindex_to_first_monthindex, &self.quarterindex_to_monthindex_count, &self.quarterindex_to_quarterindex, + &self.semesterindex_to_first_monthindex, + &self.semesterindex_to_monthindex_count, + &self.semesterindex_to_semesterindex, &self.txindex_to_height, &self.txindex_to_txindex, &self.txindex_to_input_count, @@ -1126,6 +1198,7 @@ pub struct Indexes { pub weekindex: WeekIndex, pub monthindex: MonthIndex, pub quarterindex: QuarterIndex, + pub semesterindex: SemesterIndex, pub yearindex: YearIndex, pub decadeindex: DecadeIndex, pub difficultyepoch: DifficultyEpoch, @@ -1145,6 +1218,7 @@ impl Indexes { self.weekindex = WeekIndex::from(self.dateindex); self.monthindex = MonthIndex::from(self.dateindex); self.quarterindex = QuarterIndex::from(self.monthindex); + self.semesterindex = SemesterIndex::from(self.monthindex); self.yearindex = YearIndex::from(self.monthindex); self.decadeindex = DecadeIndex::from(self.dateindex); self.difficultyepoch = DifficultyEpoch::from(self.height); diff --git a/crates/brk_computer/src/vecs/market.rs b/crates/brk_computer/src/vecs/market.rs index 98e670660..db1fcf27e 100644 --- a/crates/brk_computer/src/vecs/market.rs +++ b/crates/brk_computer/src/vecs/market.rs @@ -5,11 +5,11 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, Computation, EagerVec, Format, StoredIndex, VecIterator}; +use crate::vecs::grouped::Source; + use super::{ Indexes, fetched, - grouped::{ - ComputedRatioVecsFromDateIndex, ComputedVecsFromDateIndex, StorableVecGeneatorOptions, - }, + grouped::{ComputedRatioVecsFromDateIndex, ComputedVecsFromDateIndex, VecBuilderOptions}, indexes, transactions, }; @@ -165,8 +165,9 @@ impl Vecs { pub fn forced_import( path: &Path, version: Version, - _computation: Computation, + computation: Computation, format: Format, + indexes: &indexes::Vecs, ) -> color_eyre::Result { Ok(Self { height_to_marketcap: EagerVec::forced_import( @@ -190,1043 +191,1319 @@ impl Vecs { indexes_to_marketcap: ComputedVecsFromDateIndex::forced_import( path, "marketcap", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_ath: ComputedVecsFromDateIndex::forced_import( path, "ath", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_drawdown: ComputedVecsFromDateIndex::forced_import( path, "drawdown", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_days_since_ath: ComputedVecsFromDateIndex::forced_import( path, "days_since_ath", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_max_days_between_aths: ComputedVecsFromDateIndex::forced_import( path, "max_days_between_aths", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_max_years_between_aths: ComputedVecsFromDateIndex::forced_import( path, "max_years_between_aths", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_1w_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "1w_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_8d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "8d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_13d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "13d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_21d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "21d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_1m_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "1m_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_34d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "34d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_55d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "55d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_89d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "89d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_144d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "144d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_200d_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "200d_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_1y_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "1y_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_2y_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "2y_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_200w_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "200w_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, indexes_to_4y_sma: ComputedRatioVecsFromDateIndex::forced_import( path, "4y_sma", - true, + Source::Compute, version + VERSION + Version::ZERO, format, + computation, + indexes, + true, )?, _1d_returns: ComputedVecsFromDateIndex::forced_import( path, "1d_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1w_returns: ComputedVecsFromDateIndex::forced_import( path, "1w_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1m_returns: ComputedVecsFromDateIndex::forced_import( path, "1m_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3m_returns: ComputedVecsFromDateIndex::forced_import( path, "3m_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6m_returns: ComputedVecsFromDateIndex::forced_import( path, "6m_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1y_returns: ComputedVecsFromDateIndex::forced_import( path, "1y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _2y_returns: ComputedVecsFromDateIndex::forced_import( path, "2y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3y_returns: ComputedVecsFromDateIndex::forced_import( path, "3y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _4y_returns: ComputedVecsFromDateIndex::forced_import( path, "4y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _5y_returns: ComputedVecsFromDateIndex::forced_import( path, "5y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6y_returns: ComputedVecsFromDateIndex::forced_import( path, "6y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _8y_returns: ComputedVecsFromDateIndex::forced_import( path, "8y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _10y_returns: ComputedVecsFromDateIndex::forced_import( path, "10y_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _2y_cagr: ComputedVecsFromDateIndex::forced_import( path, "2y_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3y_cagr: ComputedVecsFromDateIndex::forced_import( path, "3y_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _4y_cagr: ComputedVecsFromDateIndex::forced_import( path, "4y_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _5y_cagr: ComputedVecsFromDateIndex::forced_import( path, "5y_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6y_cagr: ComputedVecsFromDateIndex::forced_import( path, "6y_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _8y_cagr: ComputedVecsFromDateIndex::forced_import( path, "8y_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _10y_cagr: ComputedVecsFromDateIndex::forced_import( path, "10y_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1w_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "1w_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1m_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "1m_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3m_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "3m_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6m_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "6m_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "1y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _2y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "2y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "3y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _4y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "4y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _5y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "5y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "6y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _8y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "8y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _10y_dca_returns: ComputedVecsFromDateIndex::forced_import( path, "10y_dca_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _2y_dca_cagr: ComputedVecsFromDateIndex::forced_import( path, "2y_dca_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3y_dca_cagr: ComputedVecsFromDateIndex::forced_import( path, "3y_dca_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _4y_dca_cagr: ComputedVecsFromDateIndex::forced_import( path, "4y_dca_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _5y_dca_cagr: ComputedVecsFromDateIndex::forced_import( path, "5y_dca_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6y_dca_cagr: ComputedVecsFromDateIndex::forced_import( path, "6y_dca_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _8y_dca_cagr: ComputedVecsFromDateIndex::forced_import( path, "8y_dca_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _10y_dca_cagr: ComputedVecsFromDateIndex::forced_import( path, "10y_dca_cagr", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1w_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "1w_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "1m_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "3m_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "6m_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "1y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _2y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "2y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "3y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _4y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "4y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _5y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "5y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "6y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _8y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "8y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _10y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( path, "10y_dca_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_1d_ago: ComputedVecsFromDateIndex::forced_import( path, "price_1d_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_1w_ago: ComputedVecsFromDateIndex::forced_import( path, "price_1w_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_1m_ago: ComputedVecsFromDateIndex::forced_import( path, "price_1m_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_3m_ago: ComputedVecsFromDateIndex::forced_import( path, "price_3m_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_6m_ago: ComputedVecsFromDateIndex::forced_import( path, "price_6m_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_1y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_1y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_2y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_2y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_3y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_3y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_4y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_4y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_5y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_5y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_6y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_6y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_8y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_8y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, price_10y_ago: ComputedVecsFromDateIndex::forced_import( path, "price_10y_ago", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1w_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "1w_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1m_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "1m_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3m_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "3m_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6m_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "6m_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _1y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "1y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _2y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "2y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _3y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "3y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _4y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "4y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _5y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "5y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _6y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "6y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _8y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "8y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, _10y_dca_stack: ComputedVecsFromDateIndex::forced_import( path, "10y_dca_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2025_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2025_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2024_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2024_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2023_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2023_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2022_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2022_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2021_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2021_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2020_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2020_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2019_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2019_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2018_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2018_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2017_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2017_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2016_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2016_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2015_stack: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2015_stack", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2025_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2025_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2024_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2024_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2023_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2023_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2022_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2022_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2021_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2021_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2020_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2020_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2019_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2019_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2018_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2018_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2017_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2017_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2016_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2016_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2015_avg_price: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2015_avg_price", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2025_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2025_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2024_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2024_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2023_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2023_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2022_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2022_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2021_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2021_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2020_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2020_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2019_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2019_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2018_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2018_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2017_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2017_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2016_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2016_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, dca_class_2015_returns: ComputedVecsFromDateIndex::forced_import( path, "dca_class_2015_returns", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex::forced_import( path, "200d_sma_x2_4", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex::forced_import( path, "200d_sma_x0_8", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, }) } diff --git a/crates/brk_computer/src/vecs/mining.rs b/crates/brk_computer/src/vecs/mining.rs index 0dfc6e144..0124e61c9 100644 --- a/crates/brk_computer/src/vecs/mining.rs +++ b/crates/brk_computer/src/vecs/mining.rs @@ -5,9 +5,11 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; +use crate::vecs::grouped::Source; + use super::{ Indexes, - grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions}, + grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, VecBuilderOptions}, indexes, }; @@ -24,33 +26,40 @@ impl Vecs { pub fn forced_import( path: &Path, version: Version, - _computation: Computation, + computation: Computation, format: Format, + indexes: &indexes::Vecs, ) -> color_eyre::Result { Ok(Self { indexes_to_difficulty: ComputedVecsFromHeight::forced_import( path, "difficulty", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import( path, "difficultyepoch", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import( path, "halvingepoch", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, }) } diff --git a/crates/brk_computer/src/vecs/mod.rs b/crates/brk_computer/src/vecs/mod.rs index 9bb9ad0ad..3c53ec106 100644 --- a/crates/brk_computer/src/vecs/mod.rs +++ b/crates/brk_computer/src/vecs/mod.rs @@ -20,6 +20,8 @@ pub mod transactions; pub use indexes::Indexes; use log::info; +use crate::stores::Stores; + const VERSION: Version = Version::ONE; #[derive(Clone)] @@ -58,6 +60,7 @@ impl Vecs { version + VERSION + Version::ZERO, computation, format, + &indexes, ) .unwrap() }); @@ -68,30 +71,35 @@ impl Vecs { version + VERSION + Version::ZERO, computation, format, + &indexes, )?, mining: mining::Vecs::forced_import( path, version + VERSION + Version::ZERO, computation, format, + &indexes, )?, constants: constants::Vecs::forced_import( path, version + VERSION + Version::ZERO, computation, format, + &indexes, )?, market: market::Vecs::forced_import( path, version + VERSION + Version::ZERO, computation, format, + &indexes, )?, stateful: stateful::Vecs::forced_import( path, version + VERSION + Version::ZERO, computation, format, + &indexes, fetched.as_ref(), )?, transactions: transactions::Vecs::forced_import( @@ -108,6 +116,7 @@ impl Vecs { version + VERSION + Version::ZERO, computation, format, + &indexes, fetched.as_ref(), )?, indexes, @@ -121,6 +130,7 @@ impl Vecs { starting_indexes: brk_indexer::Indexes, fetcher: Option<&mut Fetcher>, exit: &Exit, + stores: &mut Stores, ) -> color_eyre::Result<()> { info!("Computing indexes..."); let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; @@ -178,6 +188,7 @@ impl Vecs { &self.market, &mut starting_indexes, exit, + stores, )?; self.cointime.compute( diff --git a/crates/brk_computer/src/vecs/stateful/address_cohort.rs b/crates/brk_computer/src/vecs/stateful/address_cohort.rs new file mode 100644 index 000000000..68df3badd --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/address_cohort.rs @@ -0,0 +1,257 @@ +use std::{ops::Deref, path::Path}; + +use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{ + AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator, +}; + +use crate::{ + states::AddressCohortState, + vecs::{ + Indexes, fetched, + grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, market, + stateful::{ + common, + r#trait::{CohortVecs, DynCohortVecs}, + }, + }, +}; + +const VERSION: Version = Version::ZERO; + +#[derive(Clone)] +pub struct Vecs { + starting_height: Height, + + pub state: AddressCohortState, + + pub inner: common::Vecs, + + pub height_to_address_count: EagerVec, + pub indexes_to_address_count: ComputedVecsFromHeight, +} + +impl Vecs { + #[allow(clippy::too_many_arguments)] + pub fn forced_import( + path: &Path, + cohort_name: Option<&str>, + computation: Computation, + format: Format, + version: Version, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + states_path: &Path, + compute_relative_to_all: bool, + ) -> color_eyre::Result { + let compute_dollars = fetched.is_some(); + + let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}")); + + Ok(Self { + starting_height: Height::ZERO, + state: AddressCohortState::default_and_import( + states_path, + cohort_name.unwrap_or_default(), + compute_dollars, + )?, + height_to_address_count: EagerVec::forced_import( + path, + &suffix("address_count"), + version + VERSION + Version::ZERO, + format, + )?, + indexes_to_address_count: ComputedVecsFromHeight::forced_import( + path, + &suffix("address_count"), + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + inner: common::Vecs::forced_import( + path, + cohort_name, + computation, + format, + version, + indexes, + fetched, + compute_relative_to_all, + false, + )?, + }) + } +} + +impl DynCohortVecs for Vecs { + fn starting_height(&self) -> Height { + [ + self.state.height().map_or(Height::MAX, |h| h.incremented()), + self.height_to_address_count.len().into(), + self.inner.starting_height(), + ] + .into_iter() + .min() + .unwrap() + } + + fn init(&mut self, starting_height: Height) { + if starting_height > self.starting_height() { + unreachable!() + } + + self.starting_height = starting_height; + + if let Some(prev_height) = starting_height.decremented() { + self.state.address_count = *self + .height_to_address_count + .into_iter() + .unwrap_get_inner(prev_height); + } + + self.inner + .init(&mut self.starting_height, &mut self.state.inner); + } + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + self.height_to_address_count + .validate_computed_version_or_reset_file( + base_version + self.height_to_address_count.inner_version(), + )?; + + self.inner.validate_computed_versions(base_version) + } + + fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { + if self.starting_height > height { + return Ok(()); + } + + self.height_to_address_count.forced_push_at( + height, + self.state.address_count.into(), + exit, + )?; + + self.inner.forced_pushed_at(height, exit, &self.state.inner) + } + + fn compute_then_force_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + exit: &Exit, + ) -> Result<()> { + self.inner.compute_then_force_push_unrealized_states( + height, + height_price, + dateindex, + date_price, + exit, + &self.state.inner, + ) + } + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.height_to_address_count.safe_flush(exit)?; + + self.inner + .safe_flush_stateful_vecs(height, exit, &mut self.state.inner) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part1( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> { + self.indexes_to_address_count.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_address_count), + )?; + + self.inner + .compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + } + + fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + [ + self.inner.vecs(), + self.indexes_to_address_count.vecs(), + vec![&self.height_to_address_count], + ] + .concat() + } +} + +impl CohortVecs for Vecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_address_count.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_address_count) + .collect::>() + .as_slice(), + exit, + )?; + self.inner.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.inner).collect::>(), + exit, + ) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part2( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + market: &market::Vecs, + height_to_supply: &impl AnyIterableVec, + dateindex_to_supply: &impl AnyIterableVec, + height_to_realized_cap: Option<&impl AnyIterableVec>, + dateindex_to_realized_cap: Option<&impl AnyIterableVec>, + exit: &Exit, + ) -> color_eyre::Result<()> { + self.inner.compute_rest_part2( + indexer, + indexes, + fetched, + starting_indexes, + market, + height_to_supply, + dateindex_to_supply, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + } +} + +impl Deref for Vecs { + type Target = common::Vecs; + fn deref(&self) -> &Self::Target { + &self.inner + } +} diff --git a/crates/brk_computer/src/vecs/stateful/address_cohorts.rs b/crates/brk_computer/src/vecs/stateful/address_cohorts.rs new file mode 100644 index 000000000..5f0e623dc --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/address_cohorts.rs @@ -0,0 +1,550 @@ +use std::path::Path; + +use brk_core::{ + AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, GroupFilter, Height, + Result, Version, +}; +use brk_exit::Exit; +use brk_vec::{Computation, Format}; +use derive_deref::{Deref, DerefMut}; +use rayon::prelude::*; + +use crate::vecs::{ + Indexes, fetched, indexes, + stateful::{ + address_cohort, + r#trait::{CohortVecs, DynCohortVecs}, + }, +}; + +const VERSION: Version = Version::new(0); + +#[derive(Clone, Deref, DerefMut)] +pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>); + +impl Vecs { + pub fn forced_import( + path: &Path, + version: Version, + _computation: Computation, + format: Format, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + states_path: &Path, + ) -> color_eyre::Result { + Ok(Self( + AddressGroups { + amount_range: ByAmountRange { + _0sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_with_0sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1sat_to_10sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1sat_under_10sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10sats_to_100sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10sats_under_100sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100sats_to_1k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_100sats_under_1k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1k_sats_to_10k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1k_sats_under_10k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10k_sats_to_100k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10k_sats_under_100k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100k_sats_to_1m_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_100k_sats_under_1m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1m_sats_to_10m_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1m_sats_under_10m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10m_sats_to_1btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10m_sats_under_1btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1btc_to_10btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1btc_under_10btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10btc_to_100btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10btc_under_100btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100btc_to_1k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_100btc_under_1k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1k_btc_to_10k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1k_btc_under_10k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10k_btc_to_100k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10k_btc_under_100k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100k_btc_or_more: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_100k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + }, + lt_amount: ByLowerThanAmount { + _10sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_10sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_100sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_1k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_10k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_100k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1m_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_1m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10m_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_10m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_1btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_10btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_100btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_1k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_10k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_under_100k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + }, + ge_amount: ByGreatEqualAmount { + _1sat: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1sat"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_100sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100k_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_100k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1m_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10m_sats: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _100btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_100btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _1k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_1k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + _10k_btc: address_cohort::Vecs::forced_import( + path, + Some("addrs_above_10k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + )?, + }, + } + .into(), + )) + } + + pub fn compute_overlapping_vecs( + &mut self, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + let by_size_range = self.0.amount_range.as_vec(); + + [ + self.0 + .ge_amount + .as_mut_vec() + .into_iter() + .map(|(filter, vecs)| { + ( + vecs, + by_size_range + .into_iter() + .filter(|(other, _)| filter.includes(other)) + .map(|(_, v)| v) + .collect::>(), + ) + }) + .collect::>(), + self.0 + .lt_amount + .as_mut_vec() + .into_iter() + .map(|(filter, vecs)| { + ( + vecs, + by_size_range + .into_iter() + .filter(|(other, _)| filter.includes(other)) + .map(|(_, v)| v) + .collect::>(), + ) + }) + .collect::>(), + ] + .into_par_iter() + .flatten() + .try_for_each(|(vecs, stateful)| { + vecs.compute_from_stateful(starting_indexes, &stateful, exit) + }) + } + + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.as_mut_separate_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit)) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount.rs b/crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount.rs new file mode 100644 index 000000000..ee387cff6 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount.rs @@ -0,0 +1,47 @@ +use brk_core::{ByAddressType, Height}; +use brk_vec::VecIterator; +use derive_deref::{Deref, DerefMut}; + +use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount; + +#[derive(Debug, Default, Deref, DerefMut)] +pub struct AddressTypeToAddressCount(ByAddressType); + +impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount { + fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self { + if let Some(prev_height) = starting_height.decremented() { + Self(ByAddressType { + p2pk65: groups + .p2pk65 + .into_iter() + .unwrap_get_inner(prev_height) + .into(), + p2pk33: groups + .p2pk33 + .into_iter() + .unwrap_get_inner(prev_height) + .into(), + p2pkh: groups + .p2pkh + .into_iter() + .unwrap_get_inner(prev_height) + .into(), + p2sh: groups.p2sh.into_iter().unwrap_get_inner(prev_height).into(), + p2wpkh: groups + .p2wpkh + .into_iter() + .unwrap_get_inner(prev_height) + .into(), + p2wsh: groups + .p2wsh + .into_iter() + .unwrap_get_inner(prev_height) + .into(), + p2tr: groups.p2tr.into_iter().unwrap_get_inner(prev_height).into(), + p2a: groups.p2a.into_iter().unwrap_get_inner(prev_height).into(), + }) + } else { + Default::default() + } + } +} diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_height_to_addresscount.rs b/crates/brk_computer/src/vecs/stateful/addresstype_to_height_to_addresscount.rs new file mode 100644 index 000000000..98f700654 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/addresstype_to_height_to_addresscount.rs @@ -0,0 +1,43 @@ +use brk_core::{ByAddressType, Height, Result, StoredUsize}; +use brk_exit::Exit; +use brk_vec::EagerVec; +use derive_deref::{Deref, DerefMut}; + +use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount; + +#[derive(Debug, Clone, Deref, DerefMut)] +pub struct AddressTypeToHeightToAddressCount(ByAddressType>); + +impl From>> for AddressTypeToHeightToAddressCount { + fn from(value: ByAddressType>) -> Self { + Self(value) + } +} + +impl AddressTypeToHeightToAddressCount { + pub fn forced_push_at( + &mut self, + height: Height, + addresstype_to_usize: &AddressTypeToAddressCount, + exit: &Exit, + ) -> Result<()> { + self.p2pk65 + .forced_push_at(height, addresstype_to_usize.p2pk65.into(), exit)?; + self.p2pk33 + .forced_push_at(height, addresstype_to_usize.p2pk33.into(), exit)?; + self.p2pkh + .forced_push_at(height, addresstype_to_usize.p2pkh.into(), exit)?; + self.p2sh + .forced_push_at(height, addresstype_to_usize.p2sh.into(), exit)?; + self.p2wpkh + .forced_push_at(height, addresstype_to_usize.p2wpkh.into(), exit)?; + self.p2wsh + .forced_push_at(height, addresstype_to_usize.p2wsh.into(), exit)?; + self.p2tr + .forced_push_at(height, addresstype_to_usize.p2tr.into(), exit)?; + self.p2a + .forced_push_at(height, addresstype_to_usize.p2a.into(), exit)?; + + Ok(()) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_indexes_to_addresscount.rs b/crates/brk_computer/src/vecs/stateful/addresstype_to_indexes_to_addresscount.rs new file mode 100644 index 000000000..fd384f0cd --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/addresstype_to_indexes_to_addresscount.rs @@ -0,0 +1,89 @@ +use brk_core::{ByAddressType, StoredUsize}; +use brk_exit::Exit; +use brk_vec::AnyCollectableVec; +use derive_deref::{Deref, DerefMut}; + +use crate::vecs::{ + Indexes, grouped::ComputedVecsFromHeight, indexes, + stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount, +}; + +#[derive(Clone, Deref, DerefMut)] +pub struct AddressTypeToIndexesToAddressCount(ByAddressType>); + +impl From>> + for AddressTypeToIndexesToAddressCount +{ + fn from(value: ByAddressType>) -> Self { + Self(value) + } +} + +impl AddressTypeToIndexesToAddressCount { + pub fn compute( + &mut self, + // height: Height, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount, + ) -> color_eyre::Result<()> { + self.p2pk65.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pk65), + )?; + self.p2pk33.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pk33), + )?; + self.p2pkh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pkh), + )?; + self.p2sh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2sh), + )?; + self.p2wpkh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2wpkh), + )?; + self.p2wsh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2wsh), + )?; + self.p2tr.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2tr), + )?; + self.p2a.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2a), + )?; + Ok(()) + } + // + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + self.0 + .as_typed_vec() + .into_iter() + .flat_map(|(_, v)| v.vecs()) + .collect::>() + } +} diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_set.rs b/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_set.rs new file mode 100644 index 000000000..640398383 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_set.rs @@ -0,0 +1,47 @@ +use std::{collections::BTreeSet, mem}; + +use brk_core::TypeIndex; +use derive_deref::{Deref, DerefMut}; + +use super::ByAddressType; + +#[derive(Debug, Deref, DerefMut)] +pub struct AddressTypeToTypeIndexSet(ByAddressType>); + +impl AddressTypeToTypeIndexSet { + pub fn merge(mut self, mut other: Self) -> Self { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + self + } + + fn merge_(own: &mut BTreeSet, other: &mut BTreeSet) { + if own.len() >= other.len() { + own.append(other); + } else { + other.append(own); + mem::swap(own, other); + } + } +} + +impl Default for AddressTypeToTypeIndexSet { + fn default() -> Self { + Self(ByAddressType { + p2pk65: BTreeSet::default(), + p2pk33: BTreeSet::default(), + p2pkh: BTreeSet::default(), + p2sh: BTreeSet::default(), + p2wpkh: BTreeSet::default(), + p2wsh: BTreeSet::default(), + p2tr: BTreeSet::default(), + p2a: BTreeSet::default(), + }) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_tree.rs b/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_tree.rs new file mode 100644 index 000000000..02248d441 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_tree.rs @@ -0,0 +1,51 @@ +use std::{collections::BTreeMap, mem}; + +use brk_core::TypeIndex; +use derive_deref::{Deref, DerefMut}; + +use super::ByAddressType; + +#[derive(Debug, Deref, DerefMut)] +pub struct AddressTypeToTypeIndexTree(ByAddressType>); + +impl AddressTypeToTypeIndexTree { + pub fn merge(mut self, mut other: Self) -> Self { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + self + } + + fn merge_(own: &mut BTreeMap, other: &mut BTreeMap) { + if own.len() >= other.len() { + own.append(other); + } else { + other.append(own); + mem::swap(own, other); + } + } + + pub fn unwrap(self) -> ByAddressType> { + self.0 + } +} + +impl Default for AddressTypeToTypeIndexTree { + fn default() -> Self { + Self(ByAddressType { + p2pk65: BTreeMap::default(), + p2pk33: BTreeMap::default(), + p2pkh: BTreeMap::default(), + p2sh: BTreeMap::default(), + p2wpkh: BTreeMap::default(), + p2wsh: BTreeMap::default(), + p2tr: BTreeMap::default(), + p2a: BTreeMap::default(), + }) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_vec.rs b/crates/brk_computer/src/vecs/stateful/addresstype_to_vec.rs new file mode 100644 index 000000000..c629ca6c2 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/addresstype_to_vec.rs @@ -0,0 +1,21 @@ +use derive_deref::{Deref, DerefMut}; + +use super::ByAddressType; + +#[derive(Debug, Deref, DerefMut)] +pub struct AddressTypeToVec(ByAddressType>); + +impl Default for AddressTypeToVec { + fn default() -> Self { + Self(ByAddressType { + p2pk65: vec![], + p2pk33: vec![], + p2pkh: vec![], + p2sh: vec![], + p2wpkh: vec![], + p2wsh: vec![], + p2tr: vec![], + p2a: vec![], + }) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/cohort.rs b/crates/brk_computer/src/vecs/stateful/common.rs similarity index 89% rename from crates/brk_computer/src/vecs/stateful/cohort.rs rename to crates/brk_computer/src/vecs/stateful/common.rs index 356248199..98b7d15a9 100644 --- a/crates/brk_computer/src/vecs/stateful/cohort.rs +++ b/crates/brk_computer/src/vecs/stateful/common.rs @@ -1,4 +1,3 @@ -use core::panic; use std::path::Path; use brk_core::{ @@ -6,28 +5,28 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_state::CohortState; use brk_vec::{ - AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, StoredIndex, - VecIterator, + AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec, + Format, VecIterator, }; -use crate::vecs::{ - Indexes, fetched, - grouped::{ - ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex, - ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions, +use crate::{ + states::CohortState, + vecs::{ + Indexes, fetched, + grouped::{ + ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, + ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight, + Source, VecBuilderOptions, + }, + indexes, market, }, - indexes, market, }; const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - starting_height: Height, - pub state: CohortState, - // Cumulative pub height_to_realized_cap: Option>, pub height_to_supply: EagerVec, @@ -121,11 +120,11 @@ pub struct Vecs { Option>, pub indexes_to_supply_in_profit_relative_to_circulating_supply: Option>, - pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change: + pub indexes_to_net_realized_profit_and_loss_cumulative_30d_change: Option>, - pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap: + pub indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: Option>, - pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap: + pub indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: Option>, } @@ -134,12 +133,13 @@ impl Vecs { pub fn forced_import( path: &Path, cohort_name: Option<&str>, - _computation: Computation, + computation: Computation, format: Format, version: Version, + indexes: &indexes::Vecs, fetched: Option<&fetched::Vecs>, - states_path: &Path, compute_relative_to_all: bool, + ratio_extended: bool, ) -> color_eyre::Result { let compute_dollars = fetched.is_some(); @@ -147,16 +147,57 @@ impl Vecs { let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}")); - let state = CohortState::default_and_import( - states_path, - cohort_name.unwrap_or_default(), - compute_dollars, - )?; + let dateindex_to_supply_in_profit = compute_dollars.then(|| { + EagerVec::forced_import( + path, + &suffix("supply_in_profit"), + version + VERSION + Version::ZERO, + format, + ) + .unwrap() + }); + + let dateindex_to_supply_even = compute_dollars.then(|| { + EagerVec::forced_import( + path, + &suffix("supply_even"), + version + VERSION + Version::ZERO, + format, + ) + .unwrap() + }); + + let dateindex_to_supply_in_loss = compute_dollars.then(|| { + EagerVec::forced_import( + path, + &suffix("supply_in_loss"), + version + VERSION + Version::ZERO, + format, + ) + .unwrap() + }); + + let dateindex_to_unrealized_profit = compute_dollars.then(|| { + EagerVec::forced_import( + path, + &suffix("unrealized_profit"), + version + VERSION + Version::ZERO, + format, + ) + .unwrap() + }); + + let dateindex_to_unrealized_loss = compute_dollars.then(|| { + EagerVec::forced_import( + path, + &suffix("unrealized_loss"), + version + VERSION + Version::ZERO, + format, + ) + .unwrap() + }); Ok(Self { - starting_height: Height::ZERO, - state, - height_to_supply_in_profit: compute_dollars.then(|| { EagerVec::forced_import( path, @@ -166,27 +207,21 @@ impl Vecs { ) .unwrap() }), - dateindex_to_supply_in_profit: compute_dollars.then(|| { - EagerVec::forced_import( - path, - &suffix("supply_in_profit"), - version + VERSION + Version::ZERO, - format, - ) - .unwrap() - }), indexes_to_supply_in_profit: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( path, &suffix("supply_in_profit"), - false, + dateindex_to_supply_in_profit.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, ) .unwrap() }), + dateindex_to_supply_in_profit, height_to_supply_even: compute_dollars.then(|| { EagerVec::forced_import( path, @@ -196,27 +231,21 @@ impl Vecs { ) .unwrap() }), - dateindex_to_supply_even: compute_dollars.then(|| { - EagerVec::forced_import( - path, - &suffix("supply_even"), - version + VERSION + Version::ZERO, - format, - ) - .unwrap() - }), indexes_to_supply_even: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( path, &suffix("supply_even"), - false, + dateindex_to_supply_even.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, ) .unwrap() }), + dateindex_to_supply_even, height_to_supply_in_loss: compute_dollars.then(|| { EagerVec::forced_import( path, @@ -226,27 +255,21 @@ impl Vecs { ) .unwrap() }), - dateindex_to_supply_in_loss: compute_dollars.then(|| { - EagerVec::forced_import( - path, - &suffix("supply_in_loss"), - version + VERSION + Version::ZERO, - format, - ) - .unwrap() - }), indexes_to_supply_in_loss: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( path, &suffix("supply_in_loss"), - false, + dateindex_to_supply_in_loss.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, ) .unwrap() }), + dateindex_to_supply_in_loss, height_to_unrealized_profit: compute_dollars.then(|| { EagerVec::forced_import( path, @@ -256,26 +279,20 @@ impl Vecs { ) .unwrap() }), - dateindex_to_unrealized_profit: compute_dollars.then(|| { - EagerVec::forced_import( - path, - &suffix("unrealized_profit"), - version + VERSION + Version::ZERO, - format, - ) - .unwrap() - }), indexes_to_unrealized_profit: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( path, &suffix("unrealized_profit"), - false, + dateindex_to_unrealized_profit.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), + dateindex_to_unrealized_profit, height_to_unrealized_loss: compute_dollars.then(|| { EagerVec::forced_import( path, @@ -303,26 +320,20 @@ impl Vecs { ) .unwrap() }), - dateindex_to_unrealized_loss: compute_dollars.then(|| { - EagerVec::forced_import( - path, - &suffix("unrealized_loss"), - version + VERSION + Version::ZERO, - format, - ) - .unwrap() - }), indexes_to_unrealized_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( path, &suffix("unrealized_loss"), - false, + dateindex_to_unrealized_loss.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), + dateindex_to_unrealized_loss, height_to_realized_cap: compute_dollars.then(|| { EagerVec::forced_import( path, @@ -336,10 +347,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("realized_cap"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -347,10 +360,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("min_price_paid"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -358,10 +373,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("max_price_paid"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -374,7 +391,7 @@ impl Vecs { height_to_supply_value: ComputedHeightValueVecs::forced_import( path, &suffix("supply"), - false, + Source::None, version + VERSION + Version::ZERO, format, compute_dollars, @@ -382,11 +399,13 @@ impl Vecs { indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( path, &suffix("supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, )?, height_to_utxo_count: EagerVec::forced_import( path, @@ -397,19 +416,23 @@ impl Vecs { indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( path, &suffix("utxo_count"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, indexes_to_realized_price: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( path, &suffix("realized_price"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -417,9 +440,12 @@ impl Vecs { ComputedRatioVecsFromDateIndex::forced_import( path, &suffix("realized_price"), - false, + Source::None, version + VERSION + Version::ZERO, format, + computation, + indexes, + ratio_extended, ) .unwrap() }), @@ -436,10 +462,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("realized_profit"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_sum() .add_cumulative(), ) @@ -458,10 +486,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("realized_loss"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_sum() .add_cumulative(), ) @@ -471,10 +501,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("negative_realized_loss"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_sum().add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), ) .unwrap() }), @@ -491,10 +523,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("value_created"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }), @@ -502,10 +536,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("realized_value"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }), @@ -522,10 +558,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("adjusted_value_created"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }), @@ -542,10 +580,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("value_destroyed"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }), @@ -562,10 +602,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("adjusted_value_destroyed"), - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }), @@ -573,10 +615,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("realized_cap_30d_change"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -584,10 +628,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("net_realized_profit_and_loss"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_sum() .add_cumulative(), ) @@ -623,7 +669,7 @@ impl Vecs { height_to_halved_supply_value: ComputedHeightValueVecs::forced_import( path, &suffix("halved_supply"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, compute_dollars, @@ -631,11 +677,13 @@ impl Vecs { indexes_to_halved_supply: ComputedValueVecsFromDateIndex::forced_import( path, &suffix("halved_supply"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, )?, height_to_negative_unrealized_loss: compute_dollars.then(|| { EagerVec::forced_import( @@ -650,10 +698,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("negative_unrealized_loss"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -670,10 +720,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("net_unrealized_profit_and_loss"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -693,10 +745,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("net_unrealized_profit_and_loss_relative_to_market_cap"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }, @@ -705,10 +759,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("realized_profit_relative_to_realized_cap"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }), @@ -716,10 +772,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("realized_loss_relative_to_realized_cap"), - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }), @@ -728,10 +786,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("net_realized_profit_and_loss_relative_to_realized_cap"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_sum(), + computation, + indexes, + VecBuilderOptions::default().add_sum(), ) .unwrap() }, @@ -740,7 +800,7 @@ impl Vecs { ComputedHeightValueVecs::forced_import( path, &suffix("supply_even"), - false, + Source::None, version + VERSION + Version::ZERO, format, compute_dollars, @@ -751,7 +811,7 @@ impl Vecs { ComputedHeightValueVecs::forced_import( path, &suffix("supply_in_loss"), - false, + Source::None, version + VERSION + Version::ZERO, format, compute_dollars, @@ -762,7 +822,7 @@ impl Vecs { ComputedHeightValueVecs::forced_import( path, &suffix("supply_in_profit"), - false, + Source::None, version + VERSION + Version::ZERO, format, compute_dollars, @@ -800,10 +860,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("supply_even_relative_to_own_supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -811,10 +873,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("supply_in_loss_relative_to_own_supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -822,10 +886,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("supply_in_profit_relative_to_own_supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -833,10 +899,12 @@ impl Vecs { ComputedVecsFromHeight::forced_import( path, &suffix("supply_relative_to_circulating_supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -879,10 +947,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("supply_even_relative_to_circulating_supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -892,10 +962,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("supply_in_loss_relative_to_circulating_supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -905,10 +977,12 @@ impl Vecs { ComputedVecsFromDateIndex::forced_import( path, &suffix("supply_in_profit_relative_to_circulating_supply"), - true, + Source::Compute, version + VERSION + Version::ONE, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), ) .unwrap() }), @@ -927,49 +1001,59 @@ impl Vecs { indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import( path, &suffix("coinblocks_destroyed"), - true, + Source::Compute, version + VERSION + Version::TWO, format, - StorableVecGeneatorOptions::default().add_sum().add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import( path, &suffix("coindays_destroyed"), - true, + Source::Compute, version + VERSION + Version::TWO, format, - StorableVecGeneatorOptions::default().add_sum().add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, - indexes_to_cumulative_net_realized_profit_and_loss_30d_change: compute_dollars.then(|| { + indexes_to_net_realized_profit_and_loss_cumulative_30d_change: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( path, - &format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change")), - true, + &suffix("net_realized_profit_and_loss_cumulative_30d_change"), + Source::Compute, version + VERSION + Version::new(3), format, - StorableVecGeneatorOptions::default().add_last() + computation, + indexes, + VecBuilderOptions::default().add_last() ) .unwrap() }), - indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap: compute_dollars.then(|| { + indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( path, - &format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change_relative_to_realized_cap")), - true, + &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap"), + Source::Compute, version + VERSION + Version::new(3), format, - StorableVecGeneatorOptions::default().add_last() + computation, + indexes, + VecBuilderOptions::default().add_last() ) .unwrap() }), - indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap: compute_dollars.then(|| { + indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( path, - &format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change_relative_to_market_cap")), - true, + &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap"), + Source::Compute, version + VERSION + Version::new(3), format, - StorableVecGeneatorOptions::default().add_last() + computation, + indexes, + VecBuilderOptions::default().add_last() ) .unwrap() }), @@ -978,10 +1062,6 @@ impl Vecs { pub fn starting_height(&self) -> Height { [ - self.state - .price_to_amount - .height() - .map_or(usize::MAX, |h| h.incremented().unwrap_to_usize()), self.height_to_supply.len(), self.height_to_utxo_count.len(), self.height_to_realized_cap @@ -1035,25 +1115,19 @@ impl Vecs { .unwrap() } - pub fn init(&mut self, starting_height: Height) { - if starting_height > self.starting_height() { - unreachable!() - } - - self.starting_height = starting_height; - + pub fn init(&mut self, starting_height: &mut Height, state: &mut CohortState) { if let Some(prev_height) = starting_height.decremented() { - self.state.supply.value = self + state.supply.value = self .height_to_supply .into_iter() .unwrap_get_inner(prev_height); - self.state.supply.utxos = *self + state.supply.utxos = *self .height_to_utxo_count .into_iter() .unwrap_get_inner(prev_height); if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { - self.state.realized.as_mut().unwrap().cap = height_to_realized_cap + state.realized.as_mut().unwrap().cap = height_to_realized_cap .into_iter() .unwrap_get_inner(prev_height); } @@ -1286,35 +1360,33 @@ impl Vecs { Ok(()) } - pub fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { - if self.starting_height > height { - return Ok(()); - } - + pub fn forced_pushed_at( + &mut self, + height: Height, + exit: &Exit, + state: &CohortState, + ) -> Result<()> { self.height_to_supply - .forced_push_at(height, self.state.supply.value, exit)?; + .forced_push_at(height, state.supply.value, exit)?; self.height_to_utxo_count.forced_push_at( height, - StoredUsize::from(self.state.supply.utxos), + StoredUsize::from(state.supply.utxos), exit, )?; self.height_to_satblocks_destroyed.forced_push_at( height, - self.state.satblocks_destroyed, + state.satblocks_destroyed, exit, )?; - self.height_to_satdays_destroyed.forced_push_at( - height, - self.state.satdays_destroyed, - exit, - )?; + self.height_to_satdays_destroyed + .forced_push_at(height, state.satdays_destroyed, exit)?; if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { - let realized = self.state.realized.as_ref().unwrap_or_else(|| { - dbg!((&self.state.realized, &self.state.supply)); + let realized = state.realized.as_ref().unwrap_or_else(|| { + dbg!((&state.realized, &state.supply)); panic!(); }); @@ -1355,6 +1427,7 @@ impl Vecs { dateindex: Option, date_price: Option>, exit: &Exit, + state: &CohortState, ) -> Result<()> { if let Some(height_price) = height_price { self.height_to_min_price_paid @@ -1362,9 +1435,8 @@ impl Vecs { .unwrap() .forced_push_at( height, - self.state - .price_to_amount - .first_key_value() + state + .price_to_amount_first_key_value() .map(|(&dollars, _)| dollars) .unwrap_or(Dollars::NAN), exit, @@ -1374,17 +1446,15 @@ impl Vecs { .unwrap() .forced_push_at( height, - self.state - .price_to_amount - .last_key_value() + state + .price_to_amount_last_key_value() .map(|(&dollars, _)| dollars) .unwrap_or(Dollars::NAN), exit, )?; - let (height_unrealized_state, date_unrealized_state) = self - .state - .compute_unrealized_states(height_price, date_price.unwrap()); + let (height_unrealized_state, date_unrealized_state) = + state.compute_unrealized_states(height_price, date_price.unwrap()); self.height_to_supply_even .as_mut() @@ -1436,7 +1506,12 @@ impl Vecs { Ok(()) } - pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + pub fn safe_flush_stateful_vecs( + &mut self, + height: Height, + exit: &Exit, + state: &mut CohortState, + ) -> Result<()> { self.height_to_supply.safe_flush(exit)?; self.height_to_utxo_count.safe_flush(exit)?; self.height_to_satdays_destroyed.safe_flush(exit)?; @@ -1518,7 +1593,7 @@ impl Vecs { .safe_flush(exit)?; } - self.state.commit(height)?; + state.commit(height)?; Ok(()) } @@ -2550,7 +2625,7 @@ impl Vecs { }, )?; - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change .as_mut() .unwrap() .compute_all( @@ -2572,7 +2647,7 @@ impl Vecs { }, )?; - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap. + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap. as_mut() .unwrap() .compute_all( @@ -2583,14 +2658,14 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_percentage( starting_indexes.dateindex, - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(), + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(), *dateindex_to_realized_cap.as_ref().unwrap(), exit, ) }, )?; - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap. + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap. as_mut() .unwrap() .compute_all( @@ -2601,7 +2676,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_percentage( starting_indexes.dateindex, - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(), + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(), market.indexes_to_marketcap.dateindex.as_ref().unwrap(), exit, ) @@ -2935,11 +3010,11 @@ impl Vecs { .map_or(vec![], |v| v.vecs()), self.indexes_to_coinblocks_destroyed.vecs(), self.indexes_to_coindays_destroyed.vecs(), - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref() + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change.as_ref() .map_or(vec![], |v| v.vecs()), - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap.as_ref() + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap.as_ref() .map_or(vec![], |v| v.vecs()), - self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap.as_ref() + self.indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap.as_ref() .map_or(vec![], |v| v.vecs()), ] .into_iter() diff --git a/crates/brk_computer/src/vecs/stateful/height_to_addresstype_to_vec.rs b/crates/brk_computer/src/vecs/stateful/height_to_addresstype_to_vec.rs new file mode 100644 index 000000000..5e74cf426 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/height_to_addresstype_to_vec.rs @@ -0,0 +1,9 @@ +use std::collections::BTreeMap; + +use brk_core::Height; +use derive_deref::{Deref, DerefMut}; + +use crate::vecs::stateful::AddressTypeToVec; + +#[derive(Debug, Default, Deref, DerefMut)] +pub struct HeightToAddressTypeToVec(pub BTreeMap>); diff --git a/crates/brk_computer/src/vecs/stateful/mod.rs b/crates/brk_computer/src/vecs/stateful/mod.rs index 98c20d091..2e4fb4a50 100644 --- a/crates/brk_computer/src/vecs/stateful/mod.rs +++ b/crates/brk_computer/src/vecs/stateful/mod.rs @@ -1,53 +1,91 @@ -use std::{cmp::Ordering, collections::BTreeMap, path::Path, thread}; +use std::{cmp::Ordering, collections::BTreeMap, mem, ops::ControlFlow, path::Path, thread}; -use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Result, Sats, Version}; +use brk_core::{ + AddressData, ByAddressType, CheckedSub, DateIndex, Dollars, EmptyAddressData, Height, + InputIndex, OutputIndex, OutputType, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version, +}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{ AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, GenericStoredVec, - StoredIndex, StoredVec, UnsafeSlice, VecIterator, + StoredIndex, StoredVec, VecIterator, }; +use either::Either; use log::info; -use outputs::OutputCohorts; use rayon::prelude::*; -use brk_state::{ - BlockState, OutputFilter, Outputs, OutputsByDateRange, OutputsByEpoch, OutputsByFromDate, - OutputsByFromSize, OutputsBySizeRange, OutputsBySpendableType, OutputsByTerm, - OutputsByUpToDate, OutputsByUpToSize, SupplyState, Transacted, +use crate::{ + BlockState, SupplyState, Transacted, + stores::Stores, + vecs::{ + grouped::{ComputedVecsFromHeight, Source}, + market, + stateful::{ + addresstype_to_addresscount::AddressTypeToAddressCount, + addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount, + addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount, + addresstype_to_typeindex_set::AddressTypeToTypeIndexSet, r#trait::DynCohortVecs, + }, + }, }; -use crate::vecs::market; - use super::{ Indexes, fetched, - grouped::{ComputedValueVecsFromHeight, StorableVecGeneatorOptions}, + grouped::{ComputedValueVecsFromHeight, VecBuilderOptions}, indexes, transactions, }; -pub mod cohort; -mod outputs; +mod address_cohort; +mod address_cohorts; +mod addresstype_to_addresscount; +mod addresstype_to_height_to_addresscount; +mod addresstype_to_indexes_to_addresscount; +mod addresstype_to_typeindex_set; +mod addresstype_to_typeindex_tree; +mod addresstype_to_vec; +mod common; +mod height_to_addresstype_to_vec; +mod range_map; +mod r#trait; +mod utxo_cohort; +mod utxo_cohorts; +mod withaddressdatasource; -const VERSION: Version = Version::new(5); +pub use addresstype_to_typeindex_tree::*; +pub use addresstype_to_vec::*; +use height_to_addresstype_to_vec::*; +use range_map::*; +use r#trait::CohortVecs; +pub use withaddressdatasource::WithAddressDataSource; + +const VERSION: Version = Version::new(18); #[derive(Clone)] pub struct Vecs { - chain_state: StoredVec, + pub chain_state: StoredVec, - // cointime,... pub height_to_unspendable_supply: EagerVec, pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, pub height_to_opreturn_supply: EagerVec, pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight, - pub utxos_vecs: Outputs<(OutputFilter, cohort::Vecs)>, + pub addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount, + pub addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount, + pub addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount, + pub addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount, + pub utxo_vecs: utxo_cohorts::Vecs, + pub address_vecs: address_cohorts::Vecs, + + pub indexes_to_address_count: ComputedVecsFromHeight, + pub indexes_to_empty_address_count: ComputedVecsFromHeight, } impl Vecs { pub fn forced_import( path: &Path, version: Version, - _computation: Computation, + computation: Computation, format: Format, + indexes: &indexes::Vecs, fetched: Option<&fetched::Vecs>, ) -> color_eyre::Result { let compute_dollars = fetched.is_some(); @@ -74,11 +112,13 @@ impl Vecs { indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( path, "unspendable_supply", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, )?, height_to_opreturn_supply: EagerVec::forced_import( path, @@ -89,1107 +129,324 @@ impl Vecs { indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import( path, "opreturn_supply", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + VecBuilderOptions::default().add_last(), compute_dollars, + indexes, )?, - utxos_vecs: { - Outputs::<(OutputFilter, cohort::Vecs)>::from(Outputs { - all: cohort::Vecs::forced_import( + indexes_to_address_count: ComputedVecsFromHeight::forced_import( + path, + "address_count", + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import( + path, + "empty_address_count", + Source::Compute, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from( + ByAddressType { + p2pk65: EagerVec::forced_import( path, - None, - _computation, - format, + "p2pk65_address_count", version + VERSION + Version::ZERO, - fetched, - &states_path, - false, + format, )?, - by_term: OutputsByTerm { - short: cohort::Vecs::forced_import( - path, - Some("sth"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - long: cohort::Vecs::forced_import( - path, - Some("lth"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - by_up_to_date: OutputsByUpToDate { - _1d: cohort::Vecs::forced_import( - path, - Some("up_to_1d"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1w: cohort::Vecs::forced_import( - path, - Some("up_to_1w"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1m: cohort::Vecs::forced_import( - path, - Some("up_to_1m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _2m: cohort::Vecs::forced_import( - path, - Some("up_to_2m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _3m: cohort::Vecs::forced_import( - path, - Some("up_to_3m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _4m: cohort::Vecs::forced_import( - path, - Some("up_to_4m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _5m: cohort::Vecs::forced_import( - path, - Some("up_to_5m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _6m: cohort::Vecs::forced_import( - path, - Some("up_to_6m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1y: cohort::Vecs::forced_import( - path, - Some("up_to_1y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _2y: cohort::Vecs::forced_import( - path, - Some("up_to_2y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _3y: cohort::Vecs::forced_import( - path, - Some("up_to_3y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _4y: cohort::Vecs::forced_import( - path, - Some("up_to_4y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _5y: cohort::Vecs::forced_import( - path, - Some("up_to_5y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _6y: cohort::Vecs::forced_import( - path, - Some("up_to_6y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _7y: cohort::Vecs::forced_import( - path, - Some("up_to_7y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _8y: cohort::Vecs::forced_import( - path, - Some("up_to_8y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _10y: cohort::Vecs::forced_import( - path, - Some("up_to_10y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _15y: cohort::Vecs::forced_import( - path, - Some("up_to_15y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - by_from_date: OutputsByFromDate { - _1d: cohort::Vecs::forced_import( - path, - Some("from_1d"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1w: cohort::Vecs::forced_import( - path, - Some("from_1w"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1m: cohort::Vecs::forced_import( - path, - Some("from_1m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _2m: cohort::Vecs::forced_import( - path, - Some("from_2m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _3m: cohort::Vecs::forced_import( - path, - Some("from_3m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _4m: cohort::Vecs::forced_import( - path, - Some("from_4m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _5m: cohort::Vecs::forced_import( - path, - Some("from_5m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _6m: cohort::Vecs::forced_import( - path, - Some("from_6m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1y: cohort::Vecs::forced_import( - path, - Some("from_1y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _2y: cohort::Vecs::forced_import( - path, - Some("from_2y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _3y: cohort::Vecs::forced_import( - path, - Some("from_3y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _4y: cohort::Vecs::forced_import( - path, - Some("from_4y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _5y: cohort::Vecs::forced_import( - path, - Some("from_5y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _6y: cohort::Vecs::forced_import( - path, - Some("from_6y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _7y: cohort::Vecs::forced_import( - path, - Some("from_7y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _8y: cohort::Vecs::forced_import( - path, - Some("from_8y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _10y: cohort::Vecs::forced_import( - path, - Some("from_10y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _15y: cohort::Vecs::forced_import( - path, - Some("from_15y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - by_date_range: OutputsByDateRange { - start_to_1d: cohort::Vecs::forced_import( - path, - Some("start_to_1d"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1d_to_1w: cohort::Vecs::forced_import( - path, - Some("from_1d_to_1w"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1w_to_1m: cohort::Vecs::forced_import( - path, - Some("from_1w_to_1m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1m_to_2m: cohort::Vecs::forced_import( - path, - Some("from_1m_to_2m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _2m_to_3m: cohort::Vecs::forced_import( - path, - Some("from_2m_to_3m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _3m_to_4m: cohort::Vecs::forced_import( - path, - Some("from_3m_to_4m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _4m_to_5m: cohort::Vecs::forced_import( - path, - Some("from_4m_to_5m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _5m_to_6m: cohort::Vecs::forced_import( - path, - Some("from_5m_to_6m"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _6m_to_1y: cohort::Vecs::forced_import( - path, - Some("from_6m_to_1y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1y_to_2y: cohort::Vecs::forced_import( - path, - Some("from_1y_to_2y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _2y_to_3y: cohort::Vecs::forced_import( - path, - Some("from_2y_to_3y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _3y_to_4y: cohort::Vecs::forced_import( - path, - Some("from_3y_to_4y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _4y_to_5y: cohort::Vecs::forced_import( - path, - Some("from_4y_to_5y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _5y_to_6y: cohort::Vecs::forced_import( - path, - Some("from_5y_to_6y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _6y_to_7y: cohort::Vecs::forced_import( - path, - Some("from_6y_to_7y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _7y_to_8y: cohort::Vecs::forced_import( - path, - Some("from_7y_to_8y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _8y_to_10y: cohort::Vecs::forced_import( - path, - Some("from_8y_to_10y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _10y_to_15y: cohort::Vecs::forced_import( - path, - Some("from_10y_to_15y"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _15y_to_end: cohort::Vecs::forced_import( - path, - Some("from_15y_to_end"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - by_epoch: OutputsByEpoch { - _0: cohort::Vecs::forced_import( - path, - Some("epoch_0"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1: cohort::Vecs::forced_import( - path, - Some("epoch_1"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _2: cohort::Vecs::forced_import( - path, - Some("epoch_2"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _3: cohort::Vecs::forced_import( - path, - Some("epoch_3"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _4: cohort::Vecs::forced_import( - path, - Some("epoch_4"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - by_size_range: OutputsBySizeRange { - _0sats: cohort::Vecs::forced_import( - path, - Some("0sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_1sat_to_10sats: cohort::Vecs::forced_import( - path, - Some("from_1sat_to_10sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_10sats_to_100sats: cohort::Vecs::forced_import( - path, - Some("from_10sats_to_100sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_100sats_to_1_000sats: cohort::Vecs::forced_import( - path, - Some("from_100sats_to_1_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_1_000sats_to_10_000sats: cohort::Vecs::forced_import( - path, - Some("from_1_000sats_to_10_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_10_000sats_to_100_000sats: cohort::Vecs::forced_import( - path, - Some("from_10_000sats_to_100_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_100_000sats_to_1_000_000sats: cohort::Vecs::forced_import( - path, - Some("from_100_000sats_to_1_000_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_1_000_000sats_to_10_000_000sats: cohort::Vecs::forced_import( - path, - Some("from_1_000_000sats_to_10_000_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_10_000_000sats_to_1btc: cohort::Vecs::forced_import( - path, - Some("from_10_000_000sats_to_1btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_1btc_to_10btc: cohort::Vecs::forced_import( - path, - Some("from_1btc_to_10btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_10btc_to_100btc: cohort::Vecs::forced_import( - path, - Some("from_10btc_to_100btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_100btc_to_1_000btc: cohort::Vecs::forced_import( - path, - Some("from_100btc_to_1_000btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_1_000btc_to_10_000btc: cohort::Vecs::forced_import( - path, - Some("from_1_000btc_to_10_000btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_10_000btc_to_100_000btc: cohort::Vecs::forced_import( - path, - Some("from_10_000btc_to_100_000btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - from_100_000btc: cohort::Vecs::forced_import( - path, - Some("from_100_000btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - by_up_to_size: OutputsByUpToSize { - _1_000sats: cohort::Vecs::forced_import( - path, - Some("up_to_1_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _10_000sats: cohort::Vecs::forced_import( - path, - Some("up_to_10_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1btc: cohort::Vecs::forced_import( - path, - Some("up_to_1btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _10btc: cohort::Vecs::forced_import( - path, - Some("up_to_10btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _100btc: cohort::Vecs::forced_import( - path, - Some("up_to_100btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - by_from_size: OutputsByFromSize { - _1_000sats: cohort::Vecs::forced_import( - path, - Some("from_1_000sats"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _1btc: cohort::Vecs::forced_import( - path, - Some("from_1btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _10btc: cohort::Vecs::forced_import( - path, - Some("from_10btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - _100btc: cohort::Vecs::forced_import( - path, - Some("from_100btc"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - // by_value: OutputsByValue { - // up_to_1cent: cohort::Vecs::forced_import( - // path, - // Some("up_to_1cent"), - // _computation, - // format, - // fetched, - // )?, - // from_1c_to_10c: cohort::Vecs::forced_import( - // path, - // Some("from_1c_to_10c"), - // _computation, - // format, - // fetched, - // )?, - // from_10c_to_1d: cohort::Vecs::forced_import( - // path, - // Some("from_10c_to_1d"), - // _computation, - // format, - // fetched, - // )?, - // from_1d_to_10d: cohort::Vecs::forced_import( - // path, - // Some("from_1d_to_10d"), - // _computation, - // format, - // fetched, - // )?, - // from_10usd_to_100usd: cohort::Vecs::forced_import( - // path, - // Some("from_10usd_to_100usd"), - // _computation, - // format, - // fetched, - // )?, - // from_100usd_to_1_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_100usd_to_1_000usd"), - // _computation, - // format, - // fetched, - // )?, - // from_1_000usd_to_10_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_1_000usd_to_10_000usd"), - // _computation, - // format, - // fetched, - // )?, - // from_10_000usd_to_100_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_10_000usd_to_100_000usd"), - // _computation, - // format, - // fetched, - // )?, - // from_100_000usd_to_1_000_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_100_000usd_to_1_000_000usd"), - // _computation, - // format, - // fetched, - // )?, - // from_1_000_000usd_to_10_000_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_1_000_000usd_to_10_000_000usd"), - // _computation, - // format, - // fetched, - // )?, - // from_10_000_000usd_to_100_000_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_10_000_000usd_to_100_000_000usd"), - // _computation, - // format, - // fetched, - // )?, - // from_100_000_000usd_to_1_000_000_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_100_000_000usd_to_1_000_000_000usd"), - // _computation, - // format, - // fetched, - // )?, - // from_1_000_000_000usd: cohort::Vecs::forced_import( - // path, - // Some("from_1_000_000_000usd"), - // _computation, - // format, - // fetched, - // )?, - // }, - by_type: OutputsBySpendableType { - p2pk65: cohort::Vecs::forced_import( - path, - Some("p2pk65"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2pk33: cohort::Vecs::forced_import( - path, - Some("p2pk33"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2pkh: cohort::Vecs::forced_import( - path, - Some("p2pkh"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2ms: cohort::Vecs::forced_import( - path, - Some("p2ms"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2sh: cohort::Vecs::forced_import( - path, - Some("p2sh"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2wpkh: cohort::Vecs::forced_import( - path, - Some("p2wpkh"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2wsh: cohort::Vecs::forced_import( - path, - Some("p2wsh"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2tr: cohort::Vecs::forced_import( - path, - Some("p2tr"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - p2a: cohort::Vecs::forced_import( - path, - Some("p2a"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - empty: cohort::Vecs::forced_import( - path, - Some("empty"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - unknown: cohort::Vecs::forced_import( - path, - Some("unknown"), - _computation, - format, - version + VERSION + Version::ZERO, - fetched, - &states_path, - true, - )?, - }, - }) - }, + p2pk33: EagerVec::forced_import( + path, + "p2pk33_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2pkh: EagerVec::forced_import( + path, + "p2pkh_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2sh: EagerVec::forced_import( + path, + "p2sh_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2wpkh: EagerVec::forced_import( + path, + "p2wpkh_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2wsh: EagerVec::forced_import( + path, + "p2wsh_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2tr: EagerVec::forced_import( + path, + "p2tr_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2a: EagerVec::forced_import( + path, + "p2a_address_count", + version + VERSION + Version::ZERO, + format, + )?, + }, + ), + addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from( + ByAddressType { + p2pk65: EagerVec::forced_import( + path, + "p2pk65_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2pk33: EagerVec::forced_import( + path, + "p2pk33_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2pkh: EagerVec::forced_import( + path, + "p2pkh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2sh: EagerVec::forced_import( + path, + "p2sh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2wpkh: EagerVec::forced_import( + path, + "p2wpkh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2wsh: EagerVec::forced_import( + path, + "p2wsh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2tr: EagerVec::forced_import( + path, + "p2tr_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2a: EagerVec::forced_import( + path, + "p2a_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + }, + ), + addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from( + ByAddressType { + p2pk65: ComputedVecsFromHeight::forced_import( + path, + "p2pk65_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2pk33: ComputedVecsFromHeight::forced_import( + path, + "p2pk33_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2pkh: ComputedVecsFromHeight::forced_import( + path, + "p2pkh_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2sh: ComputedVecsFromHeight::forced_import( + path, + "p2sh_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2wpkh: ComputedVecsFromHeight::forced_import( + path, + "p2wpkh_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2wsh: ComputedVecsFromHeight::forced_import( + path, + "p2wsh_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2tr: ComputedVecsFromHeight::forced_import( + path, + "p2tr_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2a: ComputedVecsFromHeight::forced_import( + path, + "p2a_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + }, + ), + addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from( + ByAddressType { + p2pk65: ComputedVecsFromHeight::forced_import( + path, + "p2pk65_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2pk33: ComputedVecsFromHeight::forced_import( + path, + "p2pk33_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2pkh: ComputedVecsFromHeight::forced_import( + path, + "p2pkh_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2sh: ComputedVecsFromHeight::forced_import( + path, + "p2sh_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2wpkh: ComputedVecsFromHeight::forced_import( + path, + "p2wpkh_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2wsh: ComputedVecsFromHeight::forced_import( + path, + "p2wsh_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2tr: ComputedVecsFromHeight::forced_import( + path, + "p2tr_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + p2a: ComputedVecsFromHeight::forced_import( + path, + "p2a_empty_address_count", + Source::None, + version + VERSION + Version::ZERO, + format, + computation, + indexes, + VecBuilderOptions::default().add_last(), + )?, + }, + ), + utxo_vecs: utxo_cohorts::Vecs::forced_import( + path, + version, + computation, + format, + indexes, + fetched, + &states_path, + )?, + address_vecs: address_cohorts::Vecs::forced_import( + path, + version, + computation, + format, + indexes, + fetched, + &states_path, + )?, }) } @@ -1204,9 +461,18 @@ impl Vecs { // Must take ownership as its indexes will be updated for this specific function starting_indexes: &mut Indexes, exit: &Exit, + stores: &mut Stores, ) -> color_eyre::Result<()> { let height_to_first_outputindex = &indexer.vecs.height_to_first_outputindex; let height_to_first_inputindex = &indexer.vecs.height_to_first_inputindex; + let height_to_first_p2aaddressindex = &indexer.vecs.height_to_first_p2aaddressindex; + let height_to_first_p2pk33addressindex = &indexer.vecs.height_to_first_p2pk33addressindex; + let height_to_first_p2pk65addressindex = &indexer.vecs.height_to_first_p2pk65addressindex; + let height_to_first_p2pkhaddressindex = &indexer.vecs.height_to_first_p2pkhaddressindex; + let height_to_first_p2shaddressindex = &indexer.vecs.height_to_first_p2shaddressindex; + let height_to_first_p2traddressindex = &indexer.vecs.height_to_first_p2traddressindex; + let height_to_first_p2wpkhaddressindex = &indexer.vecs.height_to_first_p2wpkhaddressindex; + let height_to_first_p2wshaddressindex = &indexer.vecs.height_to_first_p2wshaddressindex; let height_to_output_count = transactions.indexes_to_output_count.height.unwrap_sum(); let height_to_input_count = transactions.indexes_to_input_count.height.unwrap_sum(); let inputindex_to_outputindex = &indexer.vecs.inputindex_to_outputindex; @@ -1215,6 +481,7 @@ impl Vecs { let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed; let outputindex_to_txindex = &indexes.outputindex_to_txindex; let outputindex_to_outputtype = &indexer.vecs.outputindex_to_outputtype; + let outputindex_to_typeindex = &indexer.vecs.outputindex_to_typeindex; let height_to_unclaimed_rewards = transactions .indexes_to_unclaimed_rewards .sats @@ -1231,19 +498,31 @@ impl Vecs { let dateindex_to_first_height = &indexes.dateindex_to_first_height; let dateindex_to_height_count = &indexes.dateindex_to_height_count; - let inputindex_to_outputindex_mmap = inputindex_to_outputindex.mmap().load(); - let outputindex_to_value_mmap = outputindex_to_value.mmap().load(); - let outputindex_to_outputtype_mmap = outputindex_to_outputtype.mmap().load(); - let outputindex_to_txindex_mmap = outputindex_to_txindex.mmap().load(); - let txindex_to_height_mmap = txindex_to_height.mmap().load(); + let outputindex_to_value_mmap = outputindex_to_value.create_mmap()?; + let outputindex_to_outputtype_mmap = outputindex_to_outputtype.create_mmap()?; + let outputindex_to_typeindex_mmap = outputindex_to_typeindex.create_mmap()?; + let mut inputindex_to_outputindex_iter = inputindex_to_outputindex.into_iter(); let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter(); let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); + let mut height_to_first_p2aaddressindex_iter = height_to_first_p2aaddressindex.into_iter(); + let mut height_to_first_p2pk33addressindex_iter = + height_to_first_p2pk33addressindex.into_iter(); + let mut height_to_first_p2pk65addressindex_iter = + height_to_first_p2pk65addressindex.into_iter(); + let mut height_to_first_p2pkhaddressindex_iter = + height_to_first_p2pkhaddressindex.into_iter(); + let mut height_to_first_p2shaddressindex_iter = + height_to_first_p2shaddressindex.into_iter(); + let mut height_to_first_p2traddressindex_iter = + height_to_first_p2traddressindex.into_iter(); + let mut height_to_first_p2wpkhaddressindex_iter = + height_to_first_p2wpkhaddressindex.into_iter(); + let mut height_to_first_p2wshaddressindex_iter = + height_to_first_p2wshaddressindex.into_iter(); let mut height_to_output_count_iter = height_to_output_count.into_iter(); let mut height_to_input_count_iter = height_to_input_count.into_iter(); - // let mut outputindex_to_value_iter_2 = outputindex_to_value.into_iter(); let mut height_to_close_iter = height_to_close.as_ref().map(|v| v.into_iter()); - // let mut outputindex_to_outputtype_iter_2 = outputindex_to_outputtype.into_iter(); let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter(); let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter(); let mut dateindex_to_close_iter = dateindex_to_close.as_ref().map(|v| v.into_iter()); @@ -1251,11 +530,17 @@ impl Vecs { let mut dateindex_to_first_height_iter = dateindex_to_first_height.into_iter(); let mut dateindex_to_height_count_iter = dateindex_to_height_count.into_iter(); - let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs(); - let base_version = Version::ZERO + height_to_first_outputindex.version() + height_to_first_inputindex.version() + + height_to_first_p2aaddressindex.version() + + height_to_first_p2pk33addressindex.version() + + height_to_first_p2pk65addressindex.version() + + height_to_first_p2pkhaddressindex.version() + + height_to_first_p2shaddressindex.version() + + height_to_first_p2traddressindex.version() + + height_to_first_p2wpkhaddressindex.version() + + height_to_first_p2wshaddressindex.version() + height_to_timestamp_fixed.version() + height_to_output_count.version() + height_to_input_count.version() @@ -1264,6 +549,7 @@ impl Vecs { + txindex_to_height.version() + outputindex_to_txindex.version() + outputindex_to_outputtype.version() + + outputindex_to_typeindex.version() + height_to_unclaimed_rewards.version() + height_to_close .as_ref() @@ -1273,11 +559,18 @@ impl Vecs { .map_or(Version::ZERO, |v| v.version()) + height_to_date_fixed.version() + dateindex_to_first_height.version() - + dateindex_to_height_count.version(); + + dateindex_to_height_count.version() + + stores.as_slice().into_iter().map(|s| s.version()).sum(); + + let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs(); + let mut separate_address_vecs = self.address_vecs.as_mut_separate_vecs(); separate_utxo_vecs .par_iter_mut() .try_for_each(|(_, v)| v.validate_computed_versions(base_version))?; + separate_address_vecs + .par_iter_mut() + .try_for_each(|(_, v)| v.validate_computed_versions(base_version))?; self.height_to_unspendable_supply .validate_computed_version_or_reset_file( base_version + self.height_to_unspendable_supply.inner_version(), @@ -1295,7 +588,17 @@ impl Vecs { .map(|(_, v)| v.starting_height()) .min() .unwrap_or_default() + .min( + separate_address_vecs + .par_iter_mut() + .map(|(_, v)| v.starting_height()) + .min() + .unwrap_or_default(), + ) .min(chain_state_starting_height) + .min(stores.starting_height()) + .min(Height::from(self.height_to_unspendable_supply.len())) + .min(Height::from(self.height_to_opreturn_supply.len())) .cmp(&chain_state_starting_height) { Ordering::Greater => unreachable!(), @@ -1323,11 +626,7 @@ impl Vecs { Ordering::Less => Height::ZERO, }; - let starting_height = starting_indexes - .height - .min(stateful_starting_height) - .min(Height::from(self.height_to_unspendable_supply.len())) - .min(Height::from(self.height_to_opreturn_supply.len())); + let starting_height = starting_indexes.height.min(stateful_starting_height); if starting_height.is_zero() { info!("Starting processing utxos from the start"); @@ -1336,9 +635,19 @@ impl Vecs { chain_state = vec![]; chain_state_starting_height = Height::ZERO; + stores.reset()?; + + info!("Resetting utxo price maps..."); + separate_utxo_vecs .par_iter_mut() - .try_for_each(|(_, v)| v.state.price_to_amount.reset())?; + .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; + + info!("Resetting address price maps..."); + + separate_address_vecs + .par_iter_mut() + .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; } if starting_height < Height::from(height_to_date_fixed.len()) { @@ -1348,6 +657,16 @@ impl Vecs { .par_iter_mut() .for_each(|(_, v)| v.init(starting_height)); + separate_address_vecs + .par_iter_mut() + .for_each(|(_, v)| v.init(starting_height)); + + let height_to_close_vec = + height_to_close.map(|height_to_close| height_to_close.collect().unwrap()); + + let height_to_timestamp_fixed_vec = height_to_timestamp_fixed.collect().unwrap(); + let outputindex_range_to_height = RangeMap::from(height_to_first_outputindex); + let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() { self.height_to_unspendable_supply .into_iter() @@ -1362,20 +681,38 @@ impl Vecs { } else { Sats::ZERO }; + let mut addresstype_to_address_count = AddressTypeToAddressCount::from(( + &self.addresstype_to_height_to_address_count, + starting_height, + )); + let mut addresstype_to_empty_address_count = AddressTypeToAddressCount::from(( + &self.addresstype_to_height_to_empty_address_count, + starting_height, + )); let mut height = starting_height; + let mut addresstype_to_typeindex_to_addressdata = + AddressTypeToTypeIndexTree::>::default(); + let mut addresstype_to_typeindex_to_emptyaddressdata = + AddressTypeToTypeIndexTree::>::default(); + (height.unwrap_to_usize()..height_to_date_fixed.len()) .map(Height::from) .try_for_each(|_height| -> color_eyre::Result<()> { height = _height; - self.utxos_vecs + info!("Processing chain at {height}..."); + + self.utxo_vecs .as_mut_separate_vecs() .iter_mut() .for_each(|(_, v)| v.state.reset_single_iteration_values()); - info!("Processing chain at {height}..."); + self.address_vecs + .as_mut_separate_vecs() + .iter_mut() + .for_each(|(_, v)| v.state.reset_single_iteration_values()); let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height); let price = height_to_close_iter @@ -1390,163 +727,293 @@ impl Vecs { let output_count = height_to_output_count_iter.unwrap_get_inner(height); let input_count = height_to_input_count_iter.unwrap_get_inner(height); - let (mut height_to_sent, mut received) = thread::scope(|s| { - if chain_state_starting_height <= height { - s.spawn(|| { - self.utxos_vecs - .tick_tock_next_block(&chain_state, timestamp); - }); - } - - let sent_handle = s.spawn(|| { - // Skip coinbase - (first_inputindex + 1..first_inputindex + *input_count) - .into_par_iter() - .map(InputIndex::from) - .map(|inputindex| { - let outputindex = inputindex_to_outputindex - .get_or_read(inputindex, &inputindex_to_outputindex_mmap) - .unwrap() - .unwrap() - .into_inner(); - - let value = outputindex_to_value - .get_or_read(outputindex, &outputindex_to_value_mmap) - .unwrap() - .unwrap() - .into_inner(); - - let input_type = outputindex_to_outputtype - .get_or_read(outputindex, &outputindex_to_outputtype_mmap) - .unwrap() - .unwrap() - .into_inner(); - - // dbg!(input_type); - - if input_type.is_unspendable() { - unreachable!() - } - - let input_txindex = outputindex_to_txindex - .get_or_read(outputindex, &outputindex_to_txindex_mmap) - .unwrap() - .unwrap() - .into_inner(); - - let height = txindex_to_height - .get_or_read(input_txindex, &txindex_to_height_mmap) - .unwrap() - .unwrap() - .into_inner(); - - (height, value, input_type) - }) - .fold( - BTreeMap::::default, - |mut tree, (height, value, input_type)| { - tree.entry(height).or_default().iterate(value, input_type); - tree - }, - ) - .reduce(BTreeMap::::default, |first, second| { - let (mut source, to_consume) = if first.len() > second.len() { - (first, second) - } else { - (second, first) - }; - to_consume.into_iter().for_each(|(k, v)| { - *source.entry(k).or_default() += v; - }); - source - }) - }); - - let received_handle = s.spawn(|| { - (first_outputindex..first_outputindex + *output_count) - .into_par_iter() - .map(OutputIndex::from) - .map(|outputindex| { - let value = outputindex_to_value - .get_or_read(outputindex, &outputindex_to_value_mmap) - .unwrap() - .unwrap() - .into_inner(); - - let output_type = outputindex_to_outputtype - .get_or_read(outputindex, &outputindex_to_outputtype_mmap) - .unwrap() - .unwrap() - .into_inner(); - - (value, output_type) - }) - .fold( - Transacted::default, - |mut transacted, (value, output_type)| { - transacted.iterate(value, output_type); - transacted - }, - ) - .reduce(Transacted::default, |acc, transacted| acc + transacted) - }); - - (sent_handle.join().unwrap(), received_handle.join().unwrap()) - }); - - unspendable_supply += received - .by_type - .unspendable - .as_vec() - .into_iter() - .map(|state| state.value) - .sum::() - + height_to_unclaimed_rewards_iter.unwrap_get_inner(height); - - opreturn_supply += received.by_type.unspendable.opreturn.value; - - if height == Height::new(0) { - received = Transacted::default(); - unspendable_supply += Sats::FIFTY_BTC; - } else if height == Height::new(91_842) || height == Height::new(91_880) { - // Need to destroy invalid coinbases due to duplicate txids - if height == Height::new(91_842) { - height_to_sent.entry(Height::new(91_812)).or_default() - } else { - height_to_sent.entry(Height::new(91_722)).or_default() - } - .iterate(Sats::FIFTY_BTC, OutputType::P2PK65); + let first_addressindexes: ByAddressType = ByAddressType { + p2a: height_to_first_p2aaddressindex_iter + .unwrap_get_inner(height) + .into(), + p2pk33: height_to_first_p2pk33addressindex_iter + .unwrap_get_inner(height) + .into(), + p2pk65: height_to_first_p2pk65addressindex_iter + .unwrap_get_inner(height) + .into(), + p2pkh: height_to_first_p2pkhaddressindex_iter + .unwrap_get_inner(height) + .into(), + p2sh: height_to_first_p2shaddressindex_iter + .unwrap_get_inner(height) + .into(), + p2tr: height_to_first_p2traddressindex_iter + .unwrap_get_inner(height) + .into(), + p2wpkh: height_to_first_p2wpkhaddressindex_iter + .unwrap_get_inner(height) + .into(), + p2wsh: height_to_first_p2wshaddressindex_iter + .unwrap_get_inner(height) + .into(), }; - if chain_state_starting_height <= height { + let ( + mut transacted, + addresstype_to_typedindex_to_received_data, + mut height_to_sent, + addresstype_to_typedindex_to_sent_data, + mut stored_addressindexes, + ) = thread::scope(|scope| { + scope.spawn(|| { + self.utxo_vecs.tick_tock_next_block(&chain_state, timestamp); + }); + + let received = scope.spawn(|| { + let mut transacted = Transacted::default(); + + let mut addresstype_to_typedindex_to_received_data = + AddressTypeToVec::<(TypeIndex, Sats)>::default(); + + let mut receiving_addressindexes = AddressTypeToTypeIndexSet::default(); + + let _ = (first_outputindex..first_outputindex + *output_count) + .map(OutputIndex::from) + .try_for_each(|outputindex| -> ControlFlow<()> { + let value = outputindex_to_value + .unwrap_read(outputindex, &outputindex_to_value_mmap); + + let output_type = outputindex_to_outputtype + .unwrap_read(outputindex, &outputindex_to_outputtype_mmap); + + transacted.iterate(value, output_type); + + if output_type.is_not_address() { + return ControlFlow::Continue(()); + } + + let typeindex = outputindex_to_typeindex + .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); + + receiving_addressindexes + .get_mut(output_type) + .unwrap() + .insert(typeindex); + + addresstype_to_typedindex_to_received_data + .get_mut(output_type) + .unwrap() + .push((typeindex, value)); + + ControlFlow::Continue(()) + }); + + ( + transacted, + addresstype_to_typedindex_to_received_data, + receiving_addressindexes, + ) + }); + + let mut height_to_sent = BTreeMap::::default(); + + let mut addresstype_to_typedindex_to_sent_data = + HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(); + + let mut sending_addressindexes = AddressTypeToTypeIndexSet::default(); + + // Skip coinbase + let _ = (first_inputindex + 1..first_inputindex + *input_count) + .map(InputIndex::from) + .try_for_each(|inputindex| -> ControlFlow<()> { + let outputindex = + inputindex_to_outputindex_iter.unwrap_get_inner(inputindex); + + let value = outputindex_to_value + .unwrap_read(outputindex, &outputindex_to_value_mmap); + + let input_type = outputindex_to_outputtype + .unwrap_read(outputindex, &outputindex_to_outputtype_mmap); + + let prev_height = + *outputindex_range_to_height.get(outputindex).unwrap(); + + height_to_sent + .entry(prev_height) + .or_default() + .iterate(value, input_type); + + if input_type.is_not_address() { + return ControlFlow::Continue(()); + } + + let typeindex = outputindex_to_typeindex + .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); + + sending_addressindexes + .get_mut(input_type) + .unwrap() + .insert(typeindex); + + addresstype_to_typedindex_to_sent_data + .entry(prev_height) + .or_default() + .get_mut(input_type) + .unwrap() + .push((typeindex, value)); + + ControlFlow::Continue(()) + }); + + let ( + transacted, + addresstype_to_typedindex_to_received_data, + receiving_addressindexes, + ) = received.join().unwrap(); + + let stored_addressindexes = + sending_addressindexes.merge(receiving_addressindexes); + + ( + transacted, + addresstype_to_typedindex_to_received_data, + height_to_sent, + addresstype_to_typedindex_to_sent_data, + stored_addressindexes, + ) + }); + + let mut stored_addressdata_with_source = stored_addressindexes + .into_typed_vec() + .into_par_iter() + .flat_map(|(_type, set)| { + set.into_par_iter().map(move |index| (_type, index)) + }) + .flat_map(|(_type, index)| { + if *first_addressindexes.get(_type).unwrap() <= index { + Some(( + _type, + index, + WithAddressDataSource::New(AddressData::default()), + )) + } else if addresstype_to_typeindex_to_addressdata + .get(_type) + .unwrap() + .contains_key(&index) + || addresstype_to_typeindex_to_emptyaddressdata + .get(_type) + .unwrap() + .contains_key(&index) + { + None + } else if let Some(addressdata) = + stores.get_addressdata(_type, index).unwrap() + { + Some(( + _type, + index, + WithAddressDataSource::FromAddressDataStore(addressdata), + )) + } else if let Some(emptyaddressdata) = + stores.get_emptyaddressdata(_type, index).unwrap() + { + Some(( + _type, + index, + WithAddressDataSource::FromEmptyAddressDataStore( + emptyaddressdata.into(), + ), + )) + } else { + unreachable!() + } + }) + .fold( + AddressTypeToTypeIndexTree::default, + |mut tree, (_type, index, addressdata)| { + tree.get_mut(_type).unwrap().insert(index, addressdata); + tree + }, + ) + .reduce(AddressTypeToTypeIndexTree::default, |a, b| a.merge(b)); + + thread::scope(|scope| { + scope.spawn(|| { + addresstype_to_typedindex_to_received_data.process_received( + &mut self.address_vecs, + &mut addresstype_to_typeindex_to_addressdata, + &mut addresstype_to_typeindex_to_emptyaddressdata, + price, + &mut addresstype_to_address_count, + &mut addresstype_to_empty_address_count, + &mut stored_addressdata_with_source, + ); + + addresstype_to_typedindex_to_sent_data + .process_sent( + &mut self.address_vecs, + &mut addresstype_to_typeindex_to_addressdata, + &mut addresstype_to_typeindex_to_emptyaddressdata, + price, + &mut addresstype_to_address_count, + &mut addresstype_to_empty_address_count, + height_to_close_vec.as_ref(), + &height_to_timestamp_fixed_vec, + height, + timestamp, + &mut stored_addressdata_with_source, + ) + .unwrap(); + }); + + if chain_state_starting_height > height { + dbg!(chain_state_starting_height, height); + panic!("temp, just making sure") + } + + unspendable_supply += transacted + .by_type + .unspendable + .as_vec() + .into_iter() + .map(|state| state.value) + .sum::() + + height_to_unclaimed_rewards_iter.unwrap_get_inner(height); + + opreturn_supply += transacted.by_type.unspendable.opreturn.value; + + if height == Height::new(0) { + transacted = Transacted::default(); + unspendable_supply += Sats::FIFTY_BTC; + } else if height == Height::new(91_842) || height == Height::new(91_880) { + // Need to destroy invalid coinbases due to duplicate txids + if height == Height::new(91_842) { + height_to_sent.entry(Height::new(91_812)).or_default() + } else { + height_to_sent.entry(Height::new(91_722)).or_default() + } + .iterate(Sats::FIFTY_BTC, OutputType::P2PK65); + }; // Push current block state before processing sends and receives chain_state.push(BlockState { - supply: received.spendable_supply.clone(), + supply: transacted.spendable_supply.clone(), price, timestamp, }); - self.utxos_vecs.receive(received, height, price); + self.utxo_vecs.receive(transacted, height, price); - let unsafe_chain_state = UnsafeSlice::new(&mut chain_state); + self.utxo_vecs.send(height_to_sent, &mut chain_state); + }); - height_to_sent.par_iter().for_each(|(height, sent)| unsafe { - (*unsafe_chain_state.get(height.unwrap_to_usize())).supply -= - &sent.spendable_supply; - }); - - self.utxos_vecs.send(height_to_sent, chain_state.as_slice()); - } else { - dbg!(chain_state_starting_height, height); - panic!("temp, just making sure") - } - - let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs(); + let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs(); separate_utxo_vecs .iter_mut() .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; + let mut separate_address_vecs = self.address_vecs.as_mut_separate_vecs(); + + separate_address_vecs + .iter_mut() + .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; + self.height_to_unspendable_supply.forced_push_at( height, unspendable_supply, @@ -1556,6 +1023,15 @@ impl Vecs { self.height_to_opreturn_supply .forced_push_at(height, opreturn_supply, exit)?; + self.addresstype_to_height_to_address_count.forced_push_at( + height, + &addresstype_to_address_count, + exit, + )?; + + self.addresstype_to_height_to_empty_address_count + .forced_push_at(height, &addresstype_to_empty_address_count, exit)?; + let date = height_to_date_fixed_iter.unwrap_get_inner(height); let dateindex = DateIndex::try_from(date).unwrap(); let date_first_height = @@ -1569,20 +1045,37 @@ impl Vecs { .as_mut() .map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex))); - separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| { - v.compute_then_force_push_unrealized_states( - height, - price, - is_date_last_height.then_some(dateindex), - date_price, - exit, + let dateindex = is_date_last_height.then_some(dateindex); + separate_utxo_vecs + .into_par_iter() + .map(|(_, v)| v as &mut dyn DynCohortVecs) + .chain( + separate_address_vecs + .into_par_iter() + .map(|(_, v)| v as &mut dyn DynCohortVecs), ) - })?; + .try_for_each(|v| { + v.compute_then_force_push_unrealized_states( + height, price, dateindex, date_price, exit, + ) + })?; - if height != Height::ZERO && height.unwrap_to_usize() % 20_000 == 0 { + if height != Height::ZERO && height.unwrap_to_usize() % 10_000 == 0 { info!("Flushing..."); exit.block(); self.flush_states(height, &chain_state, exit)?; + + // Maybe keep some from the end for both + let addresstype_to_typeindex_to_addressdata_to_consume = + mem::take(&mut addresstype_to_typeindex_to_addressdata); + let addresstype_to_typeindex_to_emptyaddressdata_to_consume = + mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata); + + stores.commit( + height, + addresstype_to_typeindex_to_addressdata_to_consume, + addresstype_to_typeindex_to_emptyaddressdata_to_consume, + )?; exit.release(); } @@ -1594,59 +1087,170 @@ impl Vecs { info!("Flushing..."); self.flush_states(height, &chain_state, exit)?; + stores.commit( + height, + mem::take(&mut addresstype_to_typeindex_to_addressdata), + mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), + )?; + } else { + exit.block(); } info!("Computing overlapping..."); - self.utxos_vecs - .compute_overlapping_vecs(starting_indexes, exit)?; + thread::scope(|scope| { + scope.spawn(|| { + self.utxo_vecs + .compute_overlapping_vecs(starting_indexes, exit) + .unwrap(); + }); + scope.spawn(|| { + self.address_vecs + .compute_overlapping_vecs(starting_indexes, exit) + .unwrap(); + }); + }); info!("Computing rest part 1..."); - self.utxos_vecs + self.indexes_to_address_count.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_sum_of_others( + starting_indexes.height, + &self + .addresstype_to_height_to_address_count + .as_typed_vec() + .into_iter() + .map(|(_, v)| v) + .collect::>(), + exit, + ) + }, + )?; + + self.indexes_to_empty_address_count.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_sum_of_others( + starting_indexes.height, + &self + .addresstype_to_height_to_empty_address_count + .as_typed_vec() + .into_iter() + .map(|(_, v)| v) + .collect::>(), + exit, + ) + }, + )?; + + self.addresstype_to_indexes_to_address_count.compute( + indexes, + starting_indexes, + exit, + &self.addresstype_to_height_to_address_count, + )?; + self.addresstype_to_indexes_to_empty_address_count.compute( + indexes, + starting_indexes, + exit, + &self.addresstype_to_height_to_empty_address_count, + )?; + + self.utxo_vecs .as_mut_vecs() - .par_iter_mut() - .try_for_each(|(_, v)| { - v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + .into_iter() + .map(|(_, v)| v) + .map(Either::Left) + .chain( + self.address_vecs + .as_mut_vecs() + .into_iter() + .map(|(_, v)| v) + .map(Either::Right), + ) + .collect::>>() + .into_par_iter() + .try_for_each(|either| match either { + Either::Left(v) => { + v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + } + Either::Right(v) => { + v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + } })?; info!("Computing rest part 2..."); - let height_to_supply = self.utxos_vecs.all.1.height_to_supply_value.bitcoin.clone(); + let height_to_supply = self.utxo_vecs.all.1.height_to_supply_value.bitcoin.clone(); let dateindex_to_supply = self - .utxos_vecs + .utxo_vecs .all .1 .indexes_to_supply .bitcoin .dateindex .clone(); - let height_to_realized_cap = self.utxos_vecs.all.1.height_to_realized_cap.clone(); + let height_to_realized_cap = self.utxo_vecs.all.1.height_to_realized_cap.clone(); let dateindex_to_realized_cap = self - .utxos_vecs + .utxo_vecs .all .1 .indexes_to_realized_cap .as_ref() .map(|v| v.dateindex.unwrap_last().clone()); + let dateindex_to_supply_ref = dateindex_to_supply.as_ref().unwrap(); + let height_to_realized_cap_ref = height_to_realized_cap.as_ref(); + let dateindex_to_realized_cap_ref = dateindex_to_realized_cap.as_ref(); - self.utxos_vecs + self.utxo_vecs .as_mut_vecs() - .par_iter_mut() - .try_for_each(|(_, v)| { - v.compute_rest_part2( + .into_iter() + .map(|(_, v)| v) + .map(Either::Left) + .chain( + self.address_vecs + .as_mut_vecs() + .into_iter() + .map(|(_, v)| v) + .map(Either::Right), + ) + .collect::>>() + .into_par_iter() + .try_for_each(|either| match either { + Either::Left(v) => v.compute_rest_part2( indexer, indexes, fetched, starting_indexes, market, &height_to_supply, - dateindex_to_supply.as_ref().unwrap(), - height_to_realized_cap.as_ref(), - dateindex_to_realized_cap.as_ref(), + dateindex_to_supply_ref, + height_to_realized_cap_ref, + dateindex_to_realized_cap_ref, exit, - ) + ), + Either::Right(v) => v.compute_rest_part2( + indexer, + indexes, + fetched, + starting_indexes, + market, + &height_to_supply, + dateindex_to_supply_ref, + height_to_realized_cap_ref, + dateindex_to_realized_cap_ref, + exit, + ), })?; + self.indexes_to_unspendable_supply.compute_rest( indexer, indexes, @@ -1675,12 +1279,24 @@ impl Vecs { chain_state: &[BlockState], exit: &Exit, ) -> Result<()> { - self.utxos_vecs + self.utxo_vecs + .as_mut_separate_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; + self.address_vecs .as_mut_separate_vecs() .par_iter_mut() .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; self.height_to_unspendable_supply.safe_flush(exit)?; self.height_to_opreturn_supply.safe_flush(exit)?; + self.addresstype_to_height_to_address_count + .as_mut_vec() + .into_iter() + .try_for_each(|v| v.safe_flush(exit))?; + self.addresstype_to_height_to_empty_address_count + .as_mut_vec() + .into_iter() + .try_for_each(|v| v.safe_flush(exit))?; self.chain_state.truncate_if_needed(Height::ZERO)?; chain_state.iter().for_each(|block_state| { @@ -1693,13 +1309,32 @@ impl Vecs { pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ - self.utxos_vecs + self.utxo_vecs + .vecs() + .into_iter() + .flat_map(|v| v.vecs()) + .collect::>(), + self.address_vecs .vecs() .into_iter() .flat_map(|v| v.vecs()) .collect::>(), self.indexes_to_unspendable_supply.vecs(), self.indexes_to_opreturn_supply.vecs(), + self.indexes_to_address_count.vecs(), + self.indexes_to_empty_address_count.vecs(), + self.addresstype_to_indexes_to_address_count.vecs(), + self.addresstype_to_indexes_to_empty_address_count.vecs(), + self.addresstype_to_height_to_address_count + .as_typed_vec() + .into_iter() + .map(|(_, v)| v as &dyn AnyCollectableVec) + .collect::>(), + self.addresstype_to_height_to_empty_address_count + .as_typed_vec() + .into_iter() + .map(|(_, v)| v as &dyn AnyCollectableVec) + .collect::>(), vec![ &self.height_to_unspendable_supply, &self.height_to_opreturn_supply, @@ -1710,3 +1345,204 @@ impl Vecs { .collect::>() } } + +impl AddressTypeToVec<(TypeIndex, Sats)> { + #[allow(clippy::too_many_arguments)] + fn process_received( + mut self, + vecs: &mut address_cohorts::Vecs, + addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + price: Option, + addresstype_to_address_count: &mut ByAddressType, + addresstype_to_empty_address_count: &mut ByAddressType, + stored_addressdata_with_source: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + ) { + self.into_typed_vec().into_iter().for_each(|(_type, vec)| { + vec.into_iter().for_each(|(type_index, value)| { + let mut is_new = false; + let mut from_any_empty = false; + + let addressdata_withsource = addresstype_to_typeindex_to_addressdata + .get_mut(_type) + .unwrap() + .entry(type_index) + .or_insert_with(|| { + addresstype_to_typeindex_to_emptyaddressdata + .get_mut(_type) + .unwrap() + .remove(&type_index) + .map(|ad| { + from_any_empty = true; + ad.into() + }) + .unwrap_or_else(|| { + let addressdata = stored_addressdata_with_source + .get_mut(_type) + .unwrap() + .remove(&type_index) + .unwrap(); + is_new = addressdata.is_new(); + from_any_empty = addressdata.is_from_emptyaddressdata(); + addressdata + }) + }); + + if is_new || from_any_empty { + (*addresstype_to_address_count.get_mut(_type).unwrap()) += 1; + if from_any_empty { + (*addresstype_to_empty_address_count.get_mut(_type).unwrap()) -= 1; + } + } + + let addressdata = addressdata_withsource.deref_mut(); + + let prev_amount = addressdata.amount(); + + let amount = prev_amount + value; + + if is_new + || from_any_empty + || vecs.amount_range.get_mut(amount).0.clone() + != vecs.amount_range.get_mut(prev_amount).0.clone() + { + if !is_new && !from_any_empty { + vecs.amount_range + .get_mut(prev_amount) + .1 + .state + .subtract(addressdata); + } + + addressdata.receive(value, price); + + vecs.amount_range.get_mut(amount).1.state.add(addressdata); + } else { + vecs.amount_range + .get_mut(amount) + .1 + .state + .receive(addressdata, value, price); + } + }); + }); + } +} + +impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { + #[allow(clippy::too_many_arguments)] + fn process_sent( + self, + vecs: &mut address_cohorts::Vecs, + addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + price: Option, + addresstype_to_address_count: &mut ByAddressType, + addresstype_to_empty_address_count: &mut ByAddressType, + height_to_close_vec: Option<&Vec>>, + height_to_timestamp_fixed_vec: &[Timestamp], + height: Height, + timestamp: Timestamp, + stored_addressdata_with_source: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + ) -> Result<()> { + self.0.into_iter().try_for_each(|(prev_height, mut v)| { + let prev_price = height_to_close_vec + .as_ref() + .map(|v| **v.get(prev_height.unwrap_to_usize()).unwrap()); + + let prev_timestamp = *height_to_timestamp_fixed_vec + .get(prev_height.unwrap_to_usize()) + .unwrap(); + + let blocks_old = height.unwrap_to_usize() - prev_height.unwrap_to_usize(); + + let days_old = timestamp.difference_in_days_between_float(prev_timestamp); + + let older_than_hour = timestamp + .checked_sub(prev_timestamp) + .unwrap() + .is_more_than_hour(); + + v.into_typed_vec().into_iter().try_for_each(|(_type, vec)| { + vec.into_iter().try_for_each(|(type_index, value)| { + let typeindex_to_addressdata = addresstype_to_typeindex_to_addressdata + .get_mut(_type) + .unwrap(); + + let addressdata_withsource = typeindex_to_addressdata + .entry(type_index) + .or_insert_with(|| { + stored_addressdata_with_source + .get_mut(_type) + .unwrap() + .remove(&type_index) + .unwrap() + }); + + let addressdata = addressdata_withsource.deref_mut(); + + let prev_amount = addressdata.amount(); + + let amount = prev_amount.checked_sub(value).unwrap(); + + let will_be_empty = addressdata.outputs_len - 1 == 0; + + if will_be_empty + || vecs.amount_range.get_mut(amount).0.clone() + != vecs.amount_range.get_mut(prev_amount).0.clone() + { + vecs.amount_range + .get_mut(prev_amount) + .1 + .state + .subtract(addressdata); + + addressdata.send(value, prev_price)?; + + if will_be_empty { + if amount.is_not_zero() { + unreachable!() + } + + (*addresstype_to_address_count.get_mut(_type).unwrap()) -= 1; + (*addresstype_to_empty_address_count.get_mut(_type).unwrap()) += 1; + + let addressdata = typeindex_to_addressdata.remove(&type_index).unwrap(); + + addresstype_to_typeindex_to_emptyaddressdata + .get_mut(_type) + .unwrap() + .insert(type_index, addressdata.into()); + } else { + vecs.amount_range.get_mut(amount).1.state.add(addressdata); + } + } else { + vecs.amount_range.get_mut(amount).1.state.send( + addressdata, + value, + price, + prev_price, + blocks_old, + days_old, + older_than_hour, + )?; + } + + Ok(()) + }) + }) + }) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/outputs.rs b/crates/brk_computer/src/vecs/stateful/outputs.rs deleted file mode 100644 index c4a16efe7..000000000 --- a/crates/brk_computer/src/vecs/stateful/outputs.rs +++ /dev/null @@ -1,258 +0,0 @@ -use std::{collections::BTreeMap, ops::ControlFlow}; - -use brk_core::{CheckedSub, Dollars, HalvingEpoch, Height, Result, Timestamp}; -use brk_exit::Exit; -use brk_state::{BlockState, OutputFilter, Outputs, Transacted}; -use brk_vec::StoredIndex; -use rayon::prelude::*; - -use crate::vecs::Indexes; - -use super::cohort; - -pub trait OutputCohorts { - fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp); - fn send(&mut self, height_to_sent: BTreeMap, chain_state: &[BlockState]); - fn receive(&mut self, received: Transacted, height: Height, price: Option); - fn compute_overlapping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>; -} - -impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> { - fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) { - if chain_state.is_empty() { - return; - } - - let prev_timestamp = chain_state.last().unwrap().timestamp; - - self.by_date_range - .as_mut_vec() - .into_par_iter() - .for_each(|(filter, v)| { - let state = &mut v.state; - - let _ = chain_state - .iter() - .try_for_each(|block_state| -> ControlFlow<()> { - let prev_days_old = block_state - .timestamp - .difference_in_days_between(prev_timestamp); - let days_old = block_state.timestamp.difference_in_days_between(timestamp); - - if prev_days_old == days_old { - return ControlFlow::Continue(()); - } - - let is = filter.contains(days_old); - let was = filter.contains(prev_days_old); - - if is && !was { - state.increment(&block_state.supply, block_state.price); - } else if was && !is { - state.decrement(&block_state.supply, block_state.price); - } - - ControlFlow::Continue(()) - }); - }); - } - - fn send(&mut self, height_to_sent: BTreeMap, chain_state: &[BlockState]) { - let mut time_based_vecs = self - .by_date_range - .as_mut_vec() - .into_iter() - .chain(self.by_epoch.as_mut_vec()) - .collect::>(); - - let last_timestamp = chain_state.last().unwrap().timestamp; - let current_price = chain_state.last().unwrap().price; - - // dbg!(&height_to_sent); - - height_to_sent.into_iter().for_each(|(height, sent)| { - let block_state = chain_state.get(height.unwrap_to_usize()).unwrap(); - let prev_price = block_state.price; - - let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize(); - - let days_old = block_state - .timestamp - .difference_in_days_between(last_timestamp); - - let days_old_foat = block_state - .timestamp - .difference_in_days_between_float(last_timestamp); - - let older_than_hour = - jiff::Timestamp::from(last_timestamp.checked_sub(block_state.timestamp).unwrap()) - .as_second() - >= 60 * 60; - - time_based_vecs - .iter_mut() - .filter(|(filter, _)| match filter { - OutputFilter::From(from) => *from <= days_old, - OutputFilter::To(to) => *to > days_old, - OutputFilter::Range(range) => range.contains(&days_old), - OutputFilter::Epoch(epoch) => *epoch == HalvingEpoch::from(height), - _ => unreachable!(), - }) - .for_each(|(_, vecs)| { - vecs.state.send( - &sent.spendable_supply, - current_price, - prev_price, - blocks_old, - days_old_foat, - older_than_hour, - ); - }); - - sent.by_type.spendable.as_typed_vec().into_iter().for_each( - |(output_type, supply_state)| { - self.by_type.get_mut(output_type).1.state.send( - supply_state, - current_price, - prev_price, - blocks_old, - days_old_foat, - older_than_hour, - ) - }, - ); - - sent.by_size_group - .into_iter() - .for_each(|(group, supply_state)| { - self.by_size_range.get_mut(group).1.state.send( - &supply_state, - current_price, - prev_price, - blocks_old, - days_old_foat, - older_than_hour, - ); - }); - }); - } - - fn receive(&mut self, received: Transacted, height: Height, price: Option) { - let supply_state = received.spendable_supply; - - [ - &mut self.by_date_range.start_to_1d.1, - &mut self.by_epoch.mut_vec_from_height(height).1, - ] - .into_iter() - .for_each(|v| { - v.state.receive(&supply_state, price); - }); - - self.by_type - .as_mut_vec() - .into_iter() - .for_each(|(filter, vecs)| { - let output_type = match filter { - OutputFilter::Type(output_type) => *output_type, - _ => unreachable!(), - }; - vecs.state.receive(received.by_type.get(output_type), price) - }); - - received - .by_size_group - .into_iter() - .for_each(|(group, supply_state)| { - self.by_size_range - .get_mut(group) - .1 - .state - .receive(&supply_state, price); - }); - } - - fn compute_overlapping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> { - let by_date_range = self.by_date_range.as_vec(); - let by_size_range = self.by_size_range.as_vec(); - - [ - vec![(&mut self.all.1, self.by_epoch.vecs().to_vec())], - self.by_from_date - .as_mut_vec() - .into_iter() - .map(|(filter, vecs)| { - ( - vecs, - by_date_range - .into_iter() - .filter(|(other, _)| filter.includes(other)) - .map(|(_, v)| v) - .collect::>(), - ) - }) - .collect::>(), - self.by_up_to_date - .as_mut_vec() - .into_iter() - .map(|(filter, vecs)| { - ( - vecs, - by_date_range - .into_iter() - .filter(|(other, _)| filter.includes(other)) - .map(|(_, v)| v) - .collect::>(), - ) - }) - .collect::>(), - self.by_term - .as_mut_vec() - .into_iter() - .map(|(filter, vecs)| { - ( - vecs, - by_date_range - .into_iter() - .filter(|(other, _)| filter.includes(other)) - .map(|(_, v)| v) - .collect::>(), - ) - }) - .collect::>(), - self.by_from_size - .as_mut_vec() - .into_iter() - .map(|(filter, vecs)| { - ( - vecs, - by_size_range - .into_iter() - .filter(|(other, _)| filter.includes(other)) - .map(|(_, v)| v) - .collect::>(), - ) - }) - .collect::>(), - self.by_up_to_size - .as_mut_vec() - .into_iter() - .map(|(filter, vecs)| { - ( - vecs, - by_size_range - .into_iter() - .filter(|(other, _)| filter.includes(other)) - .map(|(_, v)| v) - .collect::>(), - ) - }) - .collect::>(), - ] - .into_par_iter() - .flatten() - .try_for_each(|(vecs, stateful)| { - vecs.compute_from_stateful(starting_indexes, &stateful, exit) - }) - } -} diff --git a/crates/brk_computer/src/vecs/stateful/range_map.rs b/crates/brk_computer/src/vecs/stateful/range_map.rs new file mode 100644 index 000000000..ee52d4da1 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/range_map.rs @@ -0,0 +1,35 @@ +use std::collections::BTreeMap; + +use brk_vec::{IndexedVec, StoredIndex, StoredType}; + +#[derive(Debug)] +pub struct RangeMap(BTreeMap); + +impl RangeMap +where + I: StoredIndex, + T: StoredIndex, +{ + pub fn get(&self, key: I) -> Option<&T> { + self.0.range(..=key).next_back().map(|(&min, value)| { + if min > key { + unreachable!() + } + value + }) + } +} + +impl From<&IndexedVec> for RangeMap +where + I: StoredIndex, + T: StoredIndex + StoredType, +{ + fn from(vec: &IndexedVec) -> Self { + Self( + vec.into_iter() + .map(|(i, v)| (v.into_owned(), i)) + .collect::>(), + ) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/trait.rs b/crates/brk_computer/src/vecs/stateful/trait.rs new file mode 100644 index 000000000..db138ebed --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/trait.rs @@ -0,0 +1,63 @@ +use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyCollectableVec, AnyIterableVec}; + +use crate::vecs::{Indexes, fetched, indexes, market}; + +pub trait DynCohortVecs: Send + Sync { + fn starting_height(&self) -> Height; + + fn init(&mut self, starting_height: Height); + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>; + + fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()>; + + fn compute_then_force_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + exit: &Exit, + ) -> Result<()>; + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>; + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part1( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()>; + + fn vecs(&self) -> Vec<&dyn AnyCollectableVec>; +} + +pub trait CohortVecs: DynCohortVecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()>; + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part2( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + market: &market::Vecs, + height_to_supply: &impl AnyIterableVec, + dateindex_to_supply: &impl AnyIterableVec, + height_to_realized_cap: Option<&impl AnyIterableVec>, + dateindex_to_realized_cap: Option<&impl AnyIterableVec>, + exit: &Exit, + ) -> color_eyre::Result<()>; +} diff --git a/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs b/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs new file mode 100644 index 000000000..4058b5906 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs @@ -0,0 +1,190 @@ +use std::{ops::Deref, path::Path}; + +use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format}; + +use crate::{ + UTXOCohortState, + vecs::{ + Indexes, fetched, indexes, market, + stateful::{ + common, + r#trait::{CohortVecs, DynCohortVecs}, + }, + }, +}; + +#[derive(Clone)] +pub struct Vecs { + starting_height: Height, + + pub state: UTXOCohortState, + + inner: common::Vecs, +} + +impl Vecs { + #[allow(clippy::too_many_arguments)] + pub fn forced_import( + path: &Path, + cohort_name: Option<&str>, + computation: Computation, + format: Format, + version: Version, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + states_path: &Path, + compute_relative_to_all: bool, + ratio_extended: bool, + ) -> color_eyre::Result { + let compute_dollars = fetched.is_some(); + + Ok(Self { + starting_height: Height::ZERO, + + state: UTXOCohortState::default_and_import( + states_path, + cohort_name.unwrap_or_default(), + compute_dollars, + )?, + + inner: common::Vecs::forced_import( + path, + cohort_name, + computation, + format, + version, + indexes, + fetched, + compute_relative_to_all, + ratio_extended, + )?, + }) + } +} + +impl DynCohortVecs for Vecs { + fn starting_height(&self) -> Height { + [ + self.state.height().map_or(Height::MAX, |h| h.incremented()), + self.inner.starting_height(), + ] + .into_iter() + .min() + .unwrap() + } + + fn init(&mut self, starting_height: Height) { + if starting_height > self.starting_height() { + unreachable!() + } + + self.starting_height = starting_height; + + self.inner.init(&mut self.starting_height, &mut self.state); + } + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + self.inner.validate_computed_versions(base_version) + } + + fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { + if self.starting_height > height { + return Ok(()); + } + + self.inner.forced_pushed_at(height, exit, &self.state) + } + + fn compute_then_force_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + exit: &Exit, + ) -> Result<()> { + self.inner.compute_then_force_push_unrealized_states( + height, + height_price, + dateindex, + date_price, + exit, + &self.state, + ) + } + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.inner + .safe_flush_stateful_vecs(height, exit, &mut self.state) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part1( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> { + self.inner + .compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + } + + fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + self.inner.vecs() + } +} + +impl CohortVecs for Vecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.inner.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.inner).collect::>(), + exit, + ) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part2( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + market: &market::Vecs, + height_to_supply: &impl AnyIterableVec, + dateindex_to_supply: &impl AnyIterableVec, + height_to_realized_cap: Option<&impl AnyIterableVec>, + dateindex_to_realized_cap: Option<&impl AnyIterableVec>, + exit: &Exit, + ) -> color_eyre::Result<()> { + self.inner.compute_rest_part2( + indexer, + indexes, + fetched, + starting_indexes, + market, + height_to_supply, + dateindex_to_supply, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + } +} + +impl Deref for Vecs { + type Target = common::Vecs; + fn deref(&self) -> &Self::Target { + &self.inner + } +} diff --git a/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs b/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs new file mode 100644 index 000000000..e2e816fa5 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs @@ -0,0 +1,1708 @@ +use std::{collections::BTreeMap, ops::ControlFlow, path::Path}; + +use brk_core::{ + ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, ByMaxAge, ByMinAge, + BySpendableType, ByTerm, CheckedSub, Dollars, GroupFilter, HalvingEpoch, Height, Result, + Timestamp, UTXOGroups, Version, +}; +use brk_exit::Exit; +use brk_vec::{Computation, Format, StoredIndex}; +use derive_deref::{Deref, DerefMut}; +use rayon::prelude::*; + +use crate::{ + states::{BlockState, Transacted}, + vecs::{Indexes, fetched, indexes, stateful::r#trait::DynCohortVecs}, +}; + +use super::{r#trait::CohortVecs, utxo_cohort}; + +const VERSION: Version = Version::new(0); + +#[derive(Clone, Deref, DerefMut)] +pub struct Vecs(UTXOGroups<(GroupFilter, utxo_cohort::Vecs)>); + +impl Vecs { + pub fn forced_import( + path: &Path, + version: Version, + _computation: Computation, + format: Format, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + states_path: &Path, + ) -> color_eyre::Result { + Ok(Self( + UTXOGroups { + all: utxo_cohort::Vecs::forced_import( + path, + None, + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + false, + true, + )?, + term: ByTerm { + short: utxo_cohort::Vecs::forced_import( + path, + Some("short_term_holders"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + long: utxo_cohort::Vecs::forced_import( + path, + Some("long_term_holders"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + }, + epoch: ByEpoch { + _0: utxo_cohort::Vecs::forced_import( + path, + Some("epoch_0"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1: utxo_cohort::Vecs::forced_import( + path, + Some("epoch_1"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _2: utxo_cohort::Vecs::forced_import( + path, + Some("epoch_2"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _3: utxo_cohort::Vecs::forced_import( + path, + Some("epoch_3"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _4: utxo_cohort::Vecs::forced_import( + path, + Some("epoch_4"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + }, + _type: BySpendableType { + p2pk65: utxo_cohort::Vecs::forced_import( + path, + Some("p2pk65"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2pk33: utxo_cohort::Vecs::forced_import( + path, + Some("p2pk33"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2pkh: utxo_cohort::Vecs::forced_import( + path, + Some("p2pkh"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2sh: utxo_cohort::Vecs::forced_import( + path, + Some("p2sh"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2wpkh: utxo_cohort::Vecs::forced_import( + path, + Some("p2wpkh"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2wsh: utxo_cohort::Vecs::forced_import( + path, + Some("p2wsh"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2tr: utxo_cohort::Vecs::forced_import( + path, + Some("p2tr"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2a: utxo_cohort::Vecs::forced_import( + path, + Some("p2a"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + p2ms: utxo_cohort::Vecs::forced_import( + path, + Some("p2ms_outputs"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + empty: utxo_cohort::Vecs::forced_import( + path, + Some("empty_outputs"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + unknown: utxo_cohort::Vecs::forced_import( + path, + Some("unknown_outputs"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + }, + max_age: ByMaxAge { + _1w: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_1w_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_1m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _2m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_2m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _3m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_3m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _4m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_4m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _5m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_5m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _6m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_6m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_1y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _2y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_2y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _3y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_3y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _4y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_4y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _5y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_5y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _6y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_6y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _7y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_7y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _8y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_8y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _10y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_10y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _12y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_12y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _15y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_15y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + }, + min_age: ByMinAge { + _1d: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1d_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1w: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1w_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _2m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_2m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _3m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_3m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _4m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_4m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _5m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_5m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _6m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_6m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _2y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_2y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _3y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_3y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _4y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_4y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _5y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_5y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _6y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_6y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _7y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_7y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _8y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_8y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _10y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_10y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _12y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_12y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + }, + age_range: ByAgeRange { + up_to_1d: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_up_to_1d_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1d_to_1w: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1d_up_to_1w_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1w_to_1m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1w_up_to_1m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1m_to_2m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1m_up_to_2m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _2m_to_3m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_2m_up_to_3m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _3m_to_4m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_3m_up_to_4m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _4m_to_5m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_4m_up_to_5m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _5m_to_6m: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_5m_up_to_6m_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _6m_to_1y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_6m_up_to_1y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _1y_to_2y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_1y_up_to_2y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _2y_to_3y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_2y_up_to_3y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _3y_to_4y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_3y_up_to_4y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _4y_to_5y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_4y_up_to_5y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _5y_to_6y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_5y_up_to_6y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _6y_to_7y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_6y_up_to_7y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _7y_to_8y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_7y_up_to_8y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _8y_to_10y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_8y_up_to_10y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _10y_to_12y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_10y_up_to_12y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + _12y_to_15y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_12y_up_to_15y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + from_15y: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_at_least_15y_old"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + true, + )?, + }, + amount_range: ByAmountRange { + _0sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_with_0sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1sat_to_10sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1sat_under_10sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10sats_to_100sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10sats_under_100sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100sats_to_1k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_100sats_under_1k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1k_sats_to_10k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1k_sats_under_10k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10k_sats_to_100k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10k_sats_under_100k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100k_sats_to_1m_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_100k_sats_under_1m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1m_sats_to_10m_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1m_sats_under_10m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10m_sats_to_1btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10m_sats_under_1btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1btc_to_10btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1btc_under_10btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10btc_to_100btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10btc_under_100btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100btc_to_1k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_100btc_under_1k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1k_btc_to_10k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1k_btc_under_10k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10k_btc_to_100k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10k_btc_under_100k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100k_btc_or_more: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_100k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + }, + lt_amount: ByLowerThanAmount { + _10sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_10sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_100sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_1k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_10k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_100k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1m_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_1m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10m_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_10m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_1btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_10btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_100btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_1k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_10k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_under_100k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + }, + ge_amount: ByGreatEqualAmount { + _1sat: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1sat"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_100sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100k_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_100k_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1m_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10m_sats: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10m_sats"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _100btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_100btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _1k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_1k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + _10k_btc: utxo_cohort::Vecs::forced_import( + path, + Some("utxos_above_10k_btc"), + _computation, + format, + version + VERSION + Version::ZERO, + indexes, + fetched, + states_path, + true, + false, + )?, + }, + } + .into(), + )) + } + + pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) { + if chain_state.is_empty() { + return; + } + + let prev_timestamp = chain_state.last().unwrap().timestamp; + + let mut vecs = self + .age_range + .as_mut_vec() + .into_iter() + .map(|(filter, v)| (filter, &mut v.state)) + .collect::>(); + + let _ = chain_state + .iter() + .try_for_each(|block_state| -> ControlFlow<()> { + let prev_days_old = + prev_timestamp.difference_in_days_between(block_state.timestamp); + let days_old = timestamp.difference_in_days_between(block_state.timestamp); + + if prev_days_old == days_old { + return ControlFlow::Continue(()); + } + + vecs.iter_mut().for_each(|(filter, state)| { + let is = filter.contains(days_old); + let was = filter.contains(prev_days_old); + + if is && !was { + state.increment(&block_state.supply, block_state.price); + } else if was && !is { + state.decrement(&block_state.supply, block_state.price); + } + }); + + ControlFlow::Continue(()) + }); + } + + pub fn send( + &mut self, + height_to_sent: BTreeMap, + chain_state: &mut [BlockState], + ) { + let mut time_based_vecs = self + .0 + .age_range + .as_mut_vec() + .into_iter() + .chain(self.0.epoch.as_mut_vec()) + .collect::>(); + + let last_timestamp = chain_state.last().unwrap().timestamp; + let current_price = chain_state.last().unwrap().price; + + height_to_sent.into_iter().for_each(|(height, sent)| { + chain_state[height.unwrap_to_usize()].supply -= &sent.spendable_supply; + + let block_state = chain_state.get(height.unwrap_to_usize()).unwrap(); + let prev_price = block_state.price; + + let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize(); + + let days_old = last_timestamp.difference_in_days_between(block_state.timestamp); + let days_old_float = + last_timestamp.difference_in_days_between_float(block_state.timestamp); + + let older_than_hour = last_timestamp + .checked_sub(block_state.timestamp) + .unwrap() + .is_more_than_hour(); + + time_based_vecs + .iter_mut() + .filter(|(filter, _)| match filter { + GroupFilter::GreaterOrEqual(from) => *from <= days_old, + GroupFilter::LowerThan(to) => *to > days_old, + GroupFilter::Range(range) => range.contains(&days_old), + GroupFilter::Epoch(epoch) => *epoch == HalvingEpoch::from(height), + _ => unreachable!(), + }) + .for_each(|(_, vecs)| { + vecs.state.send( + &sent.spendable_supply, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ); + }); + + sent.by_type.spendable.as_typed_vec().into_iter().for_each( + |(output_type, supply_state)| { + self.0._type.get_mut(output_type).1.state.send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ) + }, + ); + + sent.by_size_group + .as_typed_vec() + .into_iter() + .for_each(|(group, supply_state)| { + self.0.amount_range.get_mut(group).1.state.send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ); + }); + }); + } + + pub fn receive(&mut self, received: Transacted, height: Height, price: Option) { + let supply_state = received.spendable_supply; + + [ + &mut self.0.age_range.up_to_1d.1, + &mut self.0.epoch.mut_vec_from_height(height).1, + ] + .into_iter() + .for_each(|v| { + v.state.receive(&supply_state, price); + }); + + self._type + .as_mut_vec() + .into_iter() + .for_each(|(filter, vecs)| { + let output_type = match filter { + GroupFilter::Type(output_type) => *output_type, + _ => unreachable!(), + }; + vecs.state.receive(received.by_type.get(output_type), price) + }); + + received + .by_size_group + .as_typed_vec() + .into_iter() + .for_each(|(group, supply_state)| { + self.amount_range + .get_mut(group) + .1 + .state + .receive(supply_state, price); + }); + } + + pub fn compute_overlapping_vecs( + &mut self, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + let by_date_range = self.0.age_range.as_vec(); + let by_size_range = self.0.amount_range.as_vec(); + + [ + vec![(&mut self.0.all.1, self.0.epoch.vecs().to_vec())], + self.0 + .min_age + .as_mut_vec() + .into_iter() + .map(|(filter, vecs)| { + ( + vecs, + by_date_range + .into_iter() + .filter(|(other, _)| filter.includes(other)) + .map(|(_, v)| v) + .collect::>(), + ) + }) + .collect::>(), + self.0 + .max_age + .as_mut_vec() + .into_iter() + .map(|(filter, vecs)| { + ( + vecs, + by_date_range + .into_iter() + .filter(|(other, _)| filter.includes(other)) + .map(|(_, v)| v) + .collect::>(), + ) + }) + .collect::>(), + self.0 + .term + .as_mut_vec() + .into_iter() + .map(|(filter, vecs)| { + ( + vecs, + by_date_range + .into_iter() + .filter(|(other, _)| filter.includes(other)) + .map(|(_, v)| v) + .collect::>(), + ) + }) + .collect::>(), + self.0 + .ge_amount + .as_mut_vec() + .into_iter() + .map(|(filter, vecs)| { + ( + vecs, + by_size_range + .into_iter() + .filter(|(other, _)| filter.includes(other)) + .map(|(_, v)| v) + .collect::>(), + ) + }) + .collect::>(), + self.0 + .lt_amount + .as_mut_vec() + .into_iter() + .map(|(filter, vecs)| { + ( + vecs, + by_size_range + .into_iter() + .filter(|(other, _)| filter.includes(other)) + .map(|(_, v)| v) + .collect::>(), + ) + }) + .collect::>(), + ] + .into_par_iter() + .flatten() + .try_for_each(|(vecs, stateful)| { + vecs.compute_from_stateful(starting_indexes, &stateful, exit) + }) + } + + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.as_mut_separate_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit)) + } +} diff --git a/crates/brk_computer/src/vecs/stateful/withaddressdatasource.rs b/crates/brk_computer/src/vecs/stateful/withaddressdatasource.rs new file mode 100644 index 000000000..92dcc1b56 --- /dev/null +++ b/crates/brk_computer/src/vecs/stateful/withaddressdatasource.rs @@ -0,0 +1,62 @@ +use brk_core::{AddressData, EmptyAddressData}; + +#[derive(Debug)] +pub enum WithAddressDataSource { + New(T), + FromAddressDataStore(T), + FromEmptyAddressDataStore(T), +} + +impl WithAddressDataSource { + pub fn is_new(&self) -> bool { + matches!(self, Self::New(_)) + } + + pub fn is_from_addressdata(&self) -> bool { + matches!(self, Self::FromAddressDataStore(_)) + } + + pub fn is_from_emptyaddressdata(&self) -> bool { + matches!(self, Self::FromEmptyAddressDataStore(_)) + } + + pub fn deref(&self) -> &T { + match self { + Self::New(v) => v, + Self::FromAddressDataStore(v) => v, + Self::FromEmptyAddressDataStore(v) => v, + } + } + + pub fn deref_mut(&mut self) -> &mut T { + match self { + Self::New(v) => v, + Self::FromAddressDataStore(v) => v, + Self::FromEmptyAddressDataStore(v) => v, + } + } +} + +impl From> for WithAddressDataSource { + fn from(value: WithAddressDataSource) -> Self { + match value { + WithAddressDataSource::New(v) => Self::New(v.into()), + WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()), + WithAddressDataSource::FromEmptyAddressDataStore(v) => { + Self::FromEmptyAddressDataStore(v.into()) + } + } + } +} + +impl From> for WithAddressDataSource { + fn from(value: WithAddressDataSource) -> Self { + match value { + WithAddressDataSource::New(v) => Self::New(v.into()), + WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()), + WithAddressDataSource::FromEmptyAddressDataStore(v) => { + Self::FromEmptyAddressDataStore(v.into()) + } + } + } +} diff --git a/crates/brk_computer/src/vecs/transactions.rs b/crates/brk_computer/src/vecs/transactions.rs index aeae9d3a3..d2e008f30 100644 --- a/crates/brk_computer/src/vecs/transactions.rs +++ b/crates/brk_computer/src/vecs/transactions.rs @@ -11,11 +11,13 @@ use brk_vec::{ ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator, }; +use crate::vecs::grouped::Source; + use super::{ Indexes, fetched, grouped::{ ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, - ComputedVecsFromTxindex, StorableVecGeneatorOptions, + ComputedVecsFromTxindex, VecBuilderOptions, }, indexes, }; @@ -110,13 +112,13 @@ impl Vecs { inputindex_to_outputindex_iter .next_at(index.unwrap_to_usize()) .map(|(inputindex, outputindex)| { - let outputindex = outputindex.into_inner(); + let outputindex = outputindex.into_owned(); if outputindex == OutputIndex::COINBASE { Sats::ZERO } else if let Some((_, value)) = outputindex_to_value_iter.next_at(outputindex.unwrap_to_usize()) { - value.into_inner() + value.into_owned() } else { dbg!(inputindex, outputindex); panic!() @@ -138,12 +140,12 @@ impl Vecs { txindex_to_base_size_iter .next_at(index) .map(|(_, base_size)| { - let base_size = base_size.into_inner(); + let base_size = base_size.into_owned(); let total_size = txindex_to_total_size_iter .next_at(index) .unwrap() .1 - .into_inner(); + .into_owned(); // This is the exact definition of a weight unit, as defined by BIP-141 (quote above). let wu = usize::from(base_size) * 3 + usize::from(total_size); @@ -164,7 +166,7 @@ impl Vecs { let index = index.unwrap_to_usize(); iter.next_at(index).map(|(_, weight)| { StoredUsize::from( - bitcoin::Weight::from(weight.into_inner()).to_vbytes_ceil() as usize + bitcoin::Weight::from(weight.into_owned()).to_vbytes_ceil() as usize ) }) }, @@ -182,12 +184,12 @@ impl Vecs { txindex_to_height_iter .next_at(index.unwrap_to_usize()) .map(|(_, height)| { - let height = height.into_inner(); + let height = height.into_owned(); let txindex = height_to_first_txindex_iter .next_at(height.unwrap_to_usize()) .unwrap() .1 - .into_inner(); + .into_owned(); index == txindex }) @@ -211,12 +213,12 @@ impl Vecs { txindex_to_first_inputindex_iter .next_at(txindex) .map(|(_, first_index)| { - let first_index = usize::from(first_index.into_inner()); + let first_index = usize::from(first_index.into_owned()); let count = *txindex_to_input_count_iter .next_at(txindex) .unwrap() .1 - .into_inner(); + .into_owned(); let range = first_index..first_index + count; range.into_iter().fold(Sats::ZERO, |total, inputindex| { total @@ -224,7 +226,7 @@ impl Vecs { .next_at(inputindex) .unwrap() .1 - .into_inner() + .into_owned() }) }) }, @@ -237,7 +239,8 @@ impl Vecs { // true, // version + VERSION + Version::ZERO, // format, - // StorableVecGeneatorOptions::default() + // computation, + // StorableVecGeneatorOptions::default() // .add_average() // .add_sum() // .add_cumulative(), @@ -260,12 +263,12 @@ impl Vecs { txindex_to_first_outputindex_iter .next_at(txindex) .map(|(_, first_index)| { - let first_index = usize::from(first_index.into_inner()); + let first_index = usize::from(first_index.into_owned()); let count = *txindex_to_output_count_iter .next_at(txindex) .unwrap() .1 - .into_inner(); + .into_owned(); let range = first_index..first_index + count; range.into_iter().fold(Sats::ZERO, |total, outputindex| { total @@ -273,7 +276,7 @@ impl Vecs { .next_at(outputindex) .unwrap() .1 - .into_inner() + .into_owned() }) }) }, @@ -286,7 +289,8 @@ impl Vecs { // true, // version + VERSION + Version::ZERO, // format, - // StorableVecGeneatorOptions::default() + // computation, + // StorableVecGeneatorOptions::default() // .add_average() // .add_sum() // .add_cumulative(), @@ -303,12 +307,12 @@ impl Vecs { |txindex: TxIndex, input_iter, output_iter| { let txindex = txindex.unwrap_to_usize(); input_iter.next_at(txindex).and_then(|(_, value)| { - let input = value.into_inner(); + let input = value.into_owned(); if input.is_zero() { return Some(Sats::ZERO); } output_iter.next_at(txindex).map(|(_, value)| { - let output = value.into_inner(); + let output = value.into_owned(); input.checked_sub(output).unwrap() }) }) @@ -326,9 +330,9 @@ impl Vecs { |txindex: TxIndex, fee_iter, vsize_iter| { let txindex = txindex.unwrap_to_usize(); fee_iter.next_at(txindex).and_then(|(_, value)| { - let fee = value.into_inner(); + let fee = value.into_owned(); vsize_iter.next_at(txindex).map(|(_, value)| { - let vsize = value.into_inner(); + let vsize = value.into_owned(); Feerate::from((fee, vsize)) }) }) @@ -339,10 +343,12 @@ impl Vecs { indexes_to_tx_count: ComputedVecsFromHeight::forced_import( path, "tx_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -352,10 +358,12 @@ impl Vecs { indexes_to_input_count: ComputedVecsFromTxindex::forced_import( path, "input_count", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -365,10 +373,12 @@ impl Vecs { indexes_to_output_count: ComputedVecsFromTxindex::forced_import( path, "output_count", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -378,43 +388,43 @@ impl Vecs { indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( path, "tx_v1", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v2: ComputedVecsFromHeight::forced_import( path, "tx_v2", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v3: ComputedVecsFromHeight::forced_import( path, "tx_v3", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + indexes, + VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_fee: ComputedValueVecsFromTxindex::forced_import( path, "fee", indexes, - Some(txindex_to_fee.boxed_clone()), + Source::Vec(txindex_to_fee.boxed_clone()), version + VERSION + Version::ZERO, computation, format, fetched, - StorableVecGeneatorOptions::default() + VecBuilderOptions::default() .add_sum() .add_cumulative() .add_percentiles() @@ -424,10 +434,12 @@ impl Vecs { indexes_to_feerate: ComputedVecsFromTxindex::forced_import( path, "feerate", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_percentiles() .add_minmax() .add_average(), @@ -435,10 +447,12 @@ impl Vecs { indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import( path, "tx_vsize", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_percentiles() .add_minmax() .add_average(), @@ -446,10 +460,12 @@ impl Vecs { indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import( path, "tx_weight", - false, + Source::None, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_percentiles() .add_minmax() .add_average(), @@ -457,49 +473,55 @@ impl Vecs { indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( path, "subsidy", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + VecBuilderOptions::default() .add_percentiles() .add_sum() .add_cumulative() .add_minmax() .add_average(), compute_dollars, + indexes, )?, indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import( path, "coinbase", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + VecBuilderOptions::default() .add_sum() .add_cumulative() .add_percentiles() .add_minmax() .add_average(), compute_dollars, + indexes, )?, indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import( path, "unclaimed_rewards", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() - .add_sum() - .add_cumulative(), + computation, + VecBuilderOptions::default().add_sum().add_cumulative(), compute_dollars, + indexes, )?, indexes_to_p2a_count: ComputedVecsFromHeight::forced_import( path, "p2a_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -509,10 +531,12 @@ impl Vecs { indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import( path, "p2ms_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -522,10 +546,12 @@ impl Vecs { indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import( path, "p2pk33_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -535,10 +561,12 @@ impl Vecs { indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import( path, "p2pk65_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -548,10 +576,12 @@ impl Vecs { indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import( path, "p2pkh_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -561,10 +591,12 @@ impl Vecs { indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import( path, "p2sh_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -574,10 +606,12 @@ impl Vecs { indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import( path, "p2tr_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -587,10 +621,12 @@ impl Vecs { indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import( path, "p2wpkh_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -600,10 +636,12 @@ impl Vecs { indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import( path, "p2wsh_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -613,10 +651,12 @@ impl Vecs { indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import( path, "opreturn_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -626,10 +666,12 @@ impl Vecs { indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import( path, "unknownoutput_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -639,10 +681,12 @@ impl Vecs { indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import( path, "emptyoutput_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default() + computation, + indexes, + VecBuilderOptions::default() .add_average() .add_minmax() .add_percentiles() @@ -652,10 +696,12 @@ impl Vecs { indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import( path, "exact_utxo_count", - true, + Source::Compute, version + VERSION + Version::ZERO, format, - StorableVecGeneatorOptions::default().add_last(), + computation, + indexes, + VecBuilderOptions::default().add_last(), )?, txindex_to_is_coinbase, inputindex_to_value, diff --git a/crates/brk_core/src/enums/mod.rs b/crates/brk_core/src/enums/mod.rs deleted file mode 100644 index 9198ca8e0..000000000 --- a/crates/brk_core/src/enums/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod error; -mod value; - -pub use error::*; -pub use value::*; diff --git a/crates/brk_core/src/enums/value.rs b/crates/brk_core/src/enums/value.rs deleted file mode 100644 index 7adaf7bb2..000000000 --- a/crates/brk_core/src/enums/value.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::{fmt::Debug, ops::Deref}; - -#[derive(Debug, Clone)] -pub enum Value<'a, T> { - Ref(&'a T), - Owned(T), -} - -impl Value<'_, T> -where - T: Sized + Debug + Clone, -{ - pub fn into_inner(self) -> T { - match self { - Self::Ref(t) => t.to_owned(), - Self::Owned(t) => t, - } - } -} -impl Deref for Value<'_, T> { - type Target = T; - fn deref(&self) -> &Self::Target { - match self { - Self::Ref(t) => t, - Self::Owned(t) => t, - } - } -} -impl AsRef for Value<'_, T> -where - T: Sized + Debug + Clone, -{ - fn as_ref(&self) -> &T { - match self { - Self::Ref(t) => t, - Self::Owned(t) => t, - } - } -} diff --git a/crates/brk_core/src/enums/error.rs b/crates/brk_core/src/error.rs similarity index 100% rename from crates/brk_core/src/enums/error.rs rename to crates/brk_core/src/error.rs diff --git a/crates/brk_core/src/groups/address.rs b/crates/brk_core/src/groups/address.rs new file mode 100644 index 000000000..75a193745 --- /dev/null +++ b/crates/brk_core/src/groups/address.rs @@ -0,0 +1,55 @@ +use super::{ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, GroupFilter}; + +#[derive(Default, Clone)] +pub struct AddressGroups { + pub ge_amount: ByGreatEqualAmount, + pub amount_range: ByAmountRange, + pub lt_amount: ByLowerThanAmount, +} + +impl AddressGroups { + pub fn as_mut_vecs(&mut self) -> Vec<&mut T> { + self.ge_amount + .as_mut_vec() + .into_iter() + .chain(self.amount_range.as_mut_vec()) + .chain(self.lt_amount.as_mut_vec()) + .collect::>() + } + + pub fn as_mut_separate_vecs(&mut self) -> Vec<&mut T> { + self.amount_range + .as_mut_vec() + .into_iter() + .collect::>() + } + + pub fn as_mut_overlapping_vecs(&mut self) -> Vec<&mut T> { + self.lt_amount + .as_mut_vec() + .into_iter() + .chain(self.ge_amount.as_mut_vec()) + .collect::>() + } +} + +impl AddressGroups<(GroupFilter, T)> { + pub fn vecs(&self) -> Vec<&T> { + self.amount_range + .vecs() + .into_iter() + .chain(self.lt_amount.vecs()) + .chain(self.ge_amount.vecs()) + .collect::>() + } +} + +impl From> for AddressGroups<(GroupFilter, T)> { + fn from(value: AddressGroups) -> Self { + Self { + amount_range: ByAmountRange::from(value.amount_range), + lt_amount: ByLowerThanAmount::from(value.lt_amount), + ge_amount: ByGreatEqualAmount::from(value.ge_amount), + } + } +} diff --git a/crates/brk_core/src/groups/by_address_type.rs b/crates/brk_core/src/groups/by_address_type.rs new file mode 100644 index 000000000..394dca645 --- /dev/null +++ b/crates/brk_core/src/groups/by_address_type.rs @@ -0,0 +1,169 @@ +use std::{ + mem, + ops::{Add, AddAssign}, +}; + +use super::GroupFilter; +use crate::OutputType; + +#[derive(Default, Clone, Debug)] +pub struct ByAddressType { + pub p2pk65: T, + pub p2pk33: T, + pub p2pkh: T, + pub p2sh: T, + pub p2wpkh: T, + pub p2wsh: T, + pub p2tr: T, + pub p2a: T, +} + +impl ByAddressType { + pub fn get(&self, address_type: OutputType) -> Option<&T> { + match address_type { + OutputType::P2PK65 => Some(&self.p2pk65), + OutputType::P2PK33 => Some(&self.p2pk33), + OutputType::P2PKH => Some(&self.p2pkh), + OutputType::P2SH => Some(&self.p2sh), + OutputType::P2WPKH => Some(&self.p2wpkh), + OutputType::P2WSH => Some(&self.p2wsh), + OutputType::P2TR => Some(&self.p2tr), + OutputType::P2A => Some(&self.p2a), + _ => None, + } + } + + pub fn get_mut(&mut self, address_type: OutputType) -> Option<&mut T> { + match address_type { + OutputType::P2PK65 => Some(&mut self.p2pk65), + OutputType::P2PK33 => Some(&mut self.p2pk33), + OutputType::P2PKH => Some(&mut self.p2pkh), + OutputType::P2SH => Some(&mut self.p2sh), + OutputType::P2WPKH => Some(&mut self.p2wpkh), + OutputType::P2WSH => Some(&mut self.p2wsh), + OutputType::P2TR => Some(&mut self.p2tr), + OutputType::P2A => Some(&mut self.p2a), + _ => None, + } + } + + pub fn as_mut_vec(&mut self) -> [&mut T; 8] { + [ + &mut self.p2pk65, + &mut self.p2pk33, + &mut self.p2pkh, + &mut self.p2sh, + &mut self.p2wpkh, + &mut self.p2wsh, + &mut self.p2tr, + &mut self.p2a, + ] + } + + pub fn as_typed_vec(&self) -> [(OutputType, &T); 8] { + [ + (OutputType::P2PK65, &self.p2pk65), + (OutputType::P2PK33, &self.p2pk33), + (OutputType::P2PKH, &self.p2pkh), + (OutputType::P2SH, &self.p2sh), + (OutputType::P2WPKH, &self.p2wpkh), + (OutputType::P2WSH, &self.p2wsh), + (OutputType::P2TR, &self.p2tr), + (OutputType::P2A, &self.p2a), + ] + } + + pub fn as_mut_typed_vec(&mut self) -> [(OutputType, &mut T); 8] { + [ + (OutputType::P2PK65, &mut self.p2pk65), + (OutputType::P2PK33, &mut self.p2pk33), + (OutputType::P2PKH, &mut self.p2pkh), + (OutputType::P2SH, &mut self.p2sh), + (OutputType::P2WPKH, &mut self.p2wpkh), + (OutputType::P2WSH, &mut self.p2wsh), + (OutputType::P2TR, &mut self.p2tr), + (OutputType::P2A, &mut self.p2a), + ] + } + + pub fn into_typed_vec(&mut self) -> [(OutputType, T); 8] + where + T: Default, + { + [ + (OutputType::P2PK65, mem::take(&mut self.p2pk65)), + (OutputType::P2PK33, mem::take(&mut self.p2pk33)), + (OutputType::P2PKH, mem::take(&mut self.p2pkh)), + (OutputType::P2SH, mem::take(&mut self.p2sh)), + (OutputType::P2WPKH, mem::take(&mut self.p2wpkh)), + (OutputType::P2WSH, mem::take(&mut self.p2wsh)), + (OutputType::P2TR, mem::take(&mut self.p2tr)), + (OutputType::P2A, mem::take(&mut self.p2a)), + ] + } +} + +impl ByAddressType<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 8] { + [ + &self.p2pk65.1, + &self.p2pk33.1, + &self.p2pkh.1, + &self.p2sh.1, + &self.p2wpkh.1, + &self.p2wsh.1, + &self.p2tr.1, + &self.p2a.1, + ] + } +} + +impl From> for ByAddressType<(GroupFilter, T)> { + fn from(value: ByAddressType) -> Self { + Self { + p2pk65: (GroupFilter::Type(OutputType::P2PK65), value.p2pk65), + p2pk33: (GroupFilter::Type(OutputType::P2PK33), value.p2pk33), + p2pkh: (GroupFilter::Type(OutputType::P2PKH), value.p2pkh), + p2sh: (GroupFilter::Type(OutputType::P2SH), value.p2sh), + p2wpkh: (GroupFilter::Type(OutputType::P2WPKH), value.p2wpkh), + p2wsh: (GroupFilter::Type(OutputType::P2WSH), value.p2wsh), + p2tr: (GroupFilter::Type(OutputType::P2TR), value.p2tr), + p2a: (GroupFilter::Type(OutputType::P2A), value.p2a), + } + } +} + +impl Add for ByAddressType +where + T: Add, +{ + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self { + p2pk65: self.p2pk65 + rhs.p2pk65, + p2pk33: self.p2pk33 + rhs.p2pk33, + p2pkh: self.p2pkh + rhs.p2pkh, + p2sh: self.p2sh + rhs.p2sh, + p2wpkh: self.p2wpkh + rhs.p2wpkh, + p2wsh: self.p2wsh + rhs.p2wsh, + p2tr: self.p2tr + rhs.p2tr, + p2a: self.p2a + rhs.p2a, + } + } +} + +impl AddAssign for ByAddressType +where + T: AddAssign, +{ + fn add_assign(&mut self, rhs: Self) { + self.p2pk65 += rhs.p2pk65; + self.p2pk33 += rhs.p2pk33; + self.p2pkh += rhs.p2pkh; + self.p2sh += rhs.p2sh; + self.p2wpkh += rhs.p2wpkh; + self.p2wsh += rhs.p2wsh; + self.p2tr += rhs.p2tr; + self.p2a += rhs.p2a; + } +} diff --git a/crates/brk_core/src/groups/by_age_range.rs b/crates/brk_core/src/groups/by_age_range.rs new file mode 100644 index 000000000..676cffbbb --- /dev/null +++ b/crates/brk_core/src/groups/by_age_range.rs @@ -0,0 +1,131 @@ +use super::GroupFilter; + +#[derive(Default, Clone)] +pub struct ByAgeRange { + pub up_to_1d: T, + pub _1d_to_1w: T, + pub _1w_to_1m: T, + pub _1m_to_2m: T, + pub _2m_to_3m: T, + pub _3m_to_4m: T, + pub _4m_to_5m: T, + pub _5m_to_6m: T, + pub _6m_to_1y: T, + pub _1y_to_2y: T, + pub _2y_to_3y: T, + pub _3y_to_4y: T, + pub _4y_to_5y: T, + pub _5y_to_6y: T, + pub _6y_to_7y: T, + pub _7y_to_8y: T, + pub _8y_to_10y: T, + pub _10y_to_12y: T, + pub _12y_to_15y: T, + pub from_15y: T, +} + +impl From> for ByAgeRange<(GroupFilter, T)> { + fn from(value: ByAgeRange) -> Self { + Self { + up_to_1d: (GroupFilter::LowerThan(1), value.up_to_1d), + _1d_to_1w: (GroupFilter::Range(1..7), value._1d_to_1w), + _1w_to_1m: (GroupFilter::Range(7..30), value._1w_to_1m), + _1m_to_2m: (GroupFilter::Range(30..2 * 30), value._1m_to_2m), + _2m_to_3m: (GroupFilter::Range(2 * 30..3 * 30), value._2m_to_3m), + _3m_to_4m: (GroupFilter::Range(3 * 30..4 * 30), value._3m_to_4m), + _4m_to_5m: (GroupFilter::Range(4 * 30..5 * 30), value._4m_to_5m), + _5m_to_6m: (GroupFilter::Range(5 * 30..6 * 30), value._5m_to_6m), + _6m_to_1y: (GroupFilter::Range(6 * 30..365), value._6m_to_1y), + _1y_to_2y: (GroupFilter::Range(365..2 * 365), value._1y_to_2y), + _2y_to_3y: (GroupFilter::Range(2 * 365..3 * 365), value._2y_to_3y), + _3y_to_4y: (GroupFilter::Range(3 * 365..4 * 365), value._3y_to_4y), + _4y_to_5y: (GroupFilter::Range(4 * 365..5 * 365), value._4y_to_5y), + _5y_to_6y: (GroupFilter::Range(5 * 365..6 * 365), value._5y_to_6y), + _6y_to_7y: (GroupFilter::Range(6 * 365..7 * 365), value._6y_to_7y), + _7y_to_8y: (GroupFilter::Range(7 * 365..8 * 365), value._7y_to_8y), + _8y_to_10y: (GroupFilter::Range(8 * 365..10 * 365), value._8y_to_10y), + _10y_to_12y: (GroupFilter::Range(10 * 365..12 * 365), value._10y_to_12y), + _12y_to_15y: (GroupFilter::Range(12 * 365..15 * 365), value._12y_to_15y), + from_15y: (GroupFilter::GreaterOrEqual(15 * 365), value.from_15y), + } + } +} + +impl ByAgeRange { + pub fn as_vec(&mut self) -> [&T; 20] { + [ + &self.up_to_1d, + &self._1d_to_1w, + &self._1w_to_1m, + &self._1m_to_2m, + &self._2m_to_3m, + &self._3m_to_4m, + &self._4m_to_5m, + &self._5m_to_6m, + &self._6m_to_1y, + &self._1y_to_2y, + &self._2y_to_3y, + &self._3y_to_4y, + &self._4y_to_5y, + &self._5y_to_6y, + &self._6y_to_7y, + &self._7y_to_8y, + &self._8y_to_10y, + &self._10y_to_12y, + &self._12y_to_15y, + &self.from_15y, + ] + } + + pub fn as_mut_vec(&mut self) -> [&mut T; 20] { + [ + &mut self.up_to_1d, + &mut self._1d_to_1w, + &mut self._1w_to_1m, + &mut self._1m_to_2m, + &mut self._2m_to_3m, + &mut self._3m_to_4m, + &mut self._4m_to_5m, + &mut self._5m_to_6m, + &mut self._6m_to_1y, + &mut self._1y_to_2y, + &mut self._2y_to_3y, + &mut self._3y_to_4y, + &mut self._4y_to_5y, + &mut self._5y_to_6y, + &mut self._6y_to_7y, + &mut self._7y_to_8y, + &mut self._8y_to_10y, + &mut self._10y_to_12y, + &mut self._12y_to_15y, + &mut self.from_15y, + ] + } +} + +impl ByAgeRange<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 20] { + [ + &self.up_to_1d.1, + &self._1d_to_1w.1, + &self._1w_to_1m.1, + &self._1m_to_2m.1, + &self._2m_to_3m.1, + &self._3m_to_4m.1, + &self._4m_to_5m.1, + &self._5m_to_6m.1, + &self._6m_to_1y.1, + &self._1y_to_2y.1, + &self._2y_to_3y.1, + &self._3y_to_4y.1, + &self._4y_to_5y.1, + &self._5y_to_6y.1, + &self._6y_to_7y.1, + &self._7y_to_8y.1, + &self._8y_to_10y.1, + &self._10y_to_12y.1, + &self._12y_to_15y.1, + &self.from_15y.1, + ] + } +} diff --git a/crates/brk_core/src/groups/by_amount_range.rs b/crates/brk_core/src/groups/by_amount_range.rs new file mode 100644 index 000000000..6fdd0594d --- /dev/null +++ b/crates/brk_core/src/groups/by_amount_range.rs @@ -0,0 +1,257 @@ +use std::ops::{Add, AddAssign}; + +use crate::Sats; + +use super::GroupFilter; + +#[derive(Debug, Default, Clone)] +pub struct ByAmountRange { + pub _0sats: T, + pub _1sat_to_10sats: T, + pub _10sats_to_100sats: T, + pub _100sats_to_1k_sats: T, + pub _1k_sats_to_10k_sats: T, + pub _10k_sats_to_100k_sats: T, + pub _100k_sats_to_1m_sats: T, + pub _1m_sats_to_10m_sats: T, + pub _10m_sats_to_1btc: T, + pub _1btc_to_10btc: T, + pub _10btc_to_100btc: T, + pub _100btc_to_1k_btc: T, + pub _1k_btc_to_10k_btc: T, + pub _10k_btc_to_100k_btc: T, + pub _100k_btc_or_more: T, +} + +impl From> for ByAmountRange<(GroupFilter, T)> { + fn from(value: ByAmountRange) -> Self { + #[allow(clippy::inconsistent_digit_grouping)] + Self { + _0sats: (GroupFilter::LowerThan(Sats::_1.into()), value._0sats), + _1sat_to_10sats: ( + GroupFilter::Range(Sats::_1.into()..Sats::_10.into()), + value._1sat_to_10sats, + ), + _10sats_to_100sats: ( + GroupFilter::Range(Sats::_10.into()..Sats::_100.into()), + value._10sats_to_100sats, + ), + _100sats_to_1k_sats: ( + GroupFilter::Range(Sats::_100.into()..Sats::_1K.into()), + value._100sats_to_1k_sats, + ), + _1k_sats_to_10k_sats: ( + GroupFilter::Range(Sats::_1K.into()..Sats::_10K.into()), + value._1k_sats_to_10k_sats, + ), + _10k_sats_to_100k_sats: ( + GroupFilter::Range(Sats::_10K.into()..Sats::_100K.into()), + value._10k_sats_to_100k_sats, + ), + _100k_sats_to_1m_sats: ( + GroupFilter::Range(Sats::_100K.into()..Sats::_1M.into()), + value._100k_sats_to_1m_sats, + ), + _1m_sats_to_10m_sats: ( + GroupFilter::Range(Sats::_1M.into()..Sats::_10M.into()), + value._1m_sats_to_10m_sats, + ), + _10m_sats_to_1btc: ( + GroupFilter::Range(Sats::_10M.into()..Sats::_1BTC.into()), + value._10m_sats_to_1btc, + ), + _1btc_to_10btc: ( + GroupFilter::Range(Sats::_1BTC.into()..Sats::_10BTC.into()), + value._1btc_to_10btc, + ), + _10btc_to_100btc: ( + GroupFilter::Range(Sats::_10BTC.into()..Sats::_100BTC.into()), + value._10btc_to_100btc, + ), + _100btc_to_1k_btc: ( + GroupFilter::Range(Sats::_100BTC.into()..Sats::_1K_BTC.into()), + value._100btc_to_1k_btc, + ), + _1k_btc_to_10k_btc: ( + GroupFilter::Range(Sats::_1K_BTC.into()..Sats::_10K_BTC.into()), + value._1k_btc_to_10k_btc, + ), + _10k_btc_to_100k_btc: ( + GroupFilter::Range(Sats::_10K_BTC.into()..Sats::_100K_BTC.into()), + value._10k_btc_to_100k_btc, + ), + _100k_btc_or_more: ( + GroupFilter::GreaterOrEqual(Sats::_100K_BTC.into()), + value._100k_btc_or_more, + ), + } + } +} + +impl ByAmountRange { + #[allow(clippy::inconsistent_digit_grouping)] + pub fn get_mut(&mut self, value: Sats) -> &mut T { + if value == Sats::ZERO { + &mut self._0sats + } else if value < Sats::_10 { + &mut self._1sat_to_10sats + } else if value < Sats::_100 { + &mut self._10sats_to_100sats + } else if value < Sats::_1K { + &mut self._100sats_to_1k_sats + } else if value < Sats::_10K { + &mut self._1k_sats_to_10k_sats + } else if value < Sats::_100K { + &mut self._10k_sats_to_100k_sats + } else if value < Sats::_1M { + &mut self._100k_sats_to_1m_sats + } else if value < Sats::_10M { + &mut self._1m_sats_to_10m_sats + } else if value < Sats::_1BTC { + &mut self._10m_sats_to_1btc + } else if value < Sats::_10BTC { + &mut self._1btc_to_10btc + } else if value < Sats::_100BTC { + &mut self._10btc_to_100btc + } else if value < Sats::_1K_BTC { + &mut self._100btc_to_1k_btc + } else if value < Sats::_10K_BTC { + &mut self._1k_btc_to_10k_btc + } else if value < Sats::_100K_BTC { + &mut self._10k_btc_to_100k_btc + } else { + &mut self._100k_btc_or_more + } + } + + pub fn as_vec(&self) -> [&T; 15] { + [ + &self._0sats, + &self._1sat_to_10sats, + &self._10sats_to_100sats, + &self._100sats_to_1k_sats, + &self._1k_sats_to_10k_sats, + &self._10k_sats_to_100k_sats, + &self._100k_sats_to_1m_sats, + &self._1m_sats_to_10m_sats, + &self._10m_sats_to_1btc, + &self._1btc_to_10btc, + &self._10btc_to_100btc, + &self._100btc_to_1k_btc, + &self._1k_btc_to_10k_btc, + &self._10k_btc_to_100k_btc, + &self._100k_btc_or_more, + ] + } + + pub fn as_typed_vec(&self) -> [(Sats, &T); 15] { + [ + (Sats::ZERO, &self._0sats), + (Sats::_1, &self._1sat_to_10sats), + (Sats::_10, &self._10sats_to_100sats), + (Sats::_100, &self._100sats_to_1k_sats), + (Sats::_1K, &self._1k_sats_to_10k_sats), + (Sats::_10K, &self._10k_sats_to_100k_sats), + (Sats::_100K, &self._100k_sats_to_1m_sats), + (Sats::_1M, &self._1m_sats_to_10m_sats), + (Sats::_10M, &self._10m_sats_to_1btc), + (Sats::_1BTC, &self._1btc_to_10btc), + (Sats::_10BTC, &self._10btc_to_100btc), + (Sats::_100BTC, &self._100btc_to_1k_btc), + (Sats::_1K_BTC, &self._1k_btc_to_10k_btc), + (Sats::_10K_BTC, &self._10k_btc_to_100k_btc), + (Sats::_100K_BTC, &self._100k_btc_or_more), + ] + } + + pub fn as_mut_vec(&mut self) -> [&mut T; 15] { + [ + &mut self._0sats, + &mut self._1sat_to_10sats, + &mut self._10sats_to_100sats, + &mut self._100sats_to_1k_sats, + &mut self._1k_sats_to_10k_sats, + &mut self._10k_sats_to_100k_sats, + &mut self._100k_sats_to_1m_sats, + &mut self._1m_sats_to_10m_sats, + &mut self._10m_sats_to_1btc, + &mut self._1btc_to_10btc, + &mut self._10btc_to_100btc, + &mut self._100btc_to_1k_btc, + &mut self._1k_btc_to_10k_btc, + &mut self._10k_btc_to_100k_btc, + &mut self._100k_btc_or_more, + ] + } +} + +impl ByAmountRange<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 15] { + [ + &self._0sats.1, + &self._1sat_to_10sats.1, + &self._10sats_to_100sats.1, + &self._100sats_to_1k_sats.1, + &self._1k_sats_to_10k_sats.1, + &self._10k_sats_to_100k_sats.1, + &self._100k_sats_to_1m_sats.1, + &self._1m_sats_to_10m_sats.1, + &self._10m_sats_to_1btc.1, + &self._1btc_to_10btc.1, + &self._10btc_to_100btc.1, + &self._100btc_to_1k_btc.1, + &self._1k_btc_to_10k_btc.1, + &self._10k_btc_to_100k_btc.1, + &self._100k_btc_or_more.1, + ] + } +} + +impl Add for ByAmountRange +where + T: Add, +{ + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self { + _0sats: self._0sats + rhs._0sats, + _1sat_to_10sats: self._1sat_to_10sats + rhs._1sat_to_10sats, + _10sats_to_100sats: self._10sats_to_100sats + rhs._10sats_to_100sats, + _100sats_to_1k_sats: self._100sats_to_1k_sats + rhs._100sats_to_1k_sats, + _1k_sats_to_10k_sats: self._1k_sats_to_10k_sats + rhs._1k_sats_to_10k_sats, + _10k_sats_to_100k_sats: self._10k_sats_to_100k_sats + rhs._10k_sats_to_100k_sats, + _100k_sats_to_1m_sats: self._100k_sats_to_1m_sats + rhs._100k_sats_to_1m_sats, + _1m_sats_to_10m_sats: self._1m_sats_to_10m_sats + rhs._1m_sats_to_10m_sats, + _10m_sats_to_1btc: self._10m_sats_to_1btc + rhs._10m_sats_to_1btc, + _1btc_to_10btc: self._1btc_to_10btc + rhs._1btc_to_10btc, + _10btc_to_100btc: self._10btc_to_100btc + rhs._10btc_to_100btc, + _100btc_to_1k_btc: self._100btc_to_1k_btc + rhs._100btc_to_1k_btc, + _1k_btc_to_10k_btc: self._1k_btc_to_10k_btc + rhs._1k_btc_to_10k_btc, + _10k_btc_to_100k_btc: self._10k_btc_to_100k_btc + rhs._10k_btc_to_100k_btc, + _100k_btc_or_more: self._100k_btc_or_more + rhs._100k_btc_or_more, + } + } +} + +impl AddAssign for ByAmountRange +where + T: AddAssign, +{ + fn add_assign(&mut self, rhs: Self) { + self._0sats += rhs._0sats; + self._1sat_to_10sats += rhs._1sat_to_10sats; + self._10sats_to_100sats += rhs._10sats_to_100sats; + self._100sats_to_1k_sats += rhs._100sats_to_1k_sats; + self._1k_sats_to_10k_sats += rhs._1k_sats_to_10k_sats; + self._10k_sats_to_100k_sats += rhs._10k_sats_to_100k_sats; + self._100k_sats_to_1m_sats += rhs._100k_sats_to_1m_sats; + self._1m_sats_to_10m_sats += rhs._1m_sats_to_10m_sats; + self._10m_sats_to_1btc += rhs._10m_sats_to_1btc; + self._1btc_to_10btc += rhs._1btc_to_10btc; + self._10btc_to_100btc += rhs._10btc_to_100btc; + self._100btc_to_1k_btc += rhs._100btc_to_1k_btc; + self._1k_btc_to_10k_btc += rhs._1k_btc_to_10k_btc; + self._10k_btc_to_100k_btc += rhs._10k_btc_to_100k_btc; + self._100k_btc_or_more += rhs._100k_btc_or_more; + } +} diff --git a/crates/brk_state/src/outputs/by_epoch.rs b/crates/brk_core/src/groups/by_epoch.rs similarity index 61% rename from crates/brk_state/src/outputs/by_epoch.rs rename to crates/brk_core/src/groups/by_epoch.rs index 46e511bc9..4132fdace 100644 --- a/crates/brk_state/src/outputs/by_epoch.rs +++ b/crates/brk_core/src/groups/by_epoch.rs @@ -1,9 +1,9 @@ -use brk_core::{HalvingEpoch, Height}; +use crate::{HalvingEpoch, Height}; -use super::OutputFilter; +use super::GroupFilter; #[derive(Default, Clone)] -pub struct OutputsByEpoch { +pub struct ByEpoch { pub _0: T, pub _1: T, pub _2: T, @@ -11,19 +11,19 @@ pub struct OutputsByEpoch { pub _4: T, } -impl From> for OutputsByEpoch<(OutputFilter, T)> { - fn from(value: OutputsByEpoch) -> Self { +impl From> for ByEpoch<(GroupFilter, T)> { + fn from(value: ByEpoch) -> Self { Self { - _0: (OutputFilter::Epoch(HalvingEpoch::new(0)), value._0), - _1: (OutputFilter::Epoch(HalvingEpoch::new(1)), value._1), - _2: (OutputFilter::Epoch(HalvingEpoch::new(2)), value._2), - _3: (OutputFilter::Epoch(HalvingEpoch::new(3)), value._3), - _4: (OutputFilter::Epoch(HalvingEpoch::new(4)), value._4), + _0: (GroupFilter::Epoch(HalvingEpoch::new(0)), value._0), + _1: (GroupFilter::Epoch(HalvingEpoch::new(1)), value._1), + _2: (GroupFilter::Epoch(HalvingEpoch::new(2)), value._2), + _3: (GroupFilter::Epoch(HalvingEpoch::new(3)), value._3), + _4: (GroupFilter::Epoch(HalvingEpoch::new(4)), value._4), } } } -impl OutputsByEpoch { +impl ByEpoch { pub fn as_mut_vec(&mut self) -> [&mut T; 5] { [ &mut self._0, @@ -52,7 +52,7 @@ impl OutputsByEpoch { } } -impl OutputsByEpoch<(OutputFilter, T)> { +impl ByEpoch<(GroupFilter, T)> { pub fn vecs(&self) -> [&T; 5] { [&self._0.1, &self._1.1, &self._2.1, &self._3.1, &self._4.1] } diff --git a/crates/brk_core/src/groups/by_ge_amount.rs b/crates/brk_core/src/groups/by_ge_amount.rs new file mode 100644 index 000000000..d358dec17 --- /dev/null +++ b/crates/brk_core/src/groups/by_ge_amount.rs @@ -0,0 +1,110 @@ +use crate::Sats; + +use super::GroupFilter; + +#[derive(Default, Clone)] +pub struct ByGreatEqualAmount { + pub _1sat: T, + pub _10sats: T, + pub _100sats: T, + pub _1k_sats: T, + pub _10k_sats: T, + pub _100k_sats: T, + pub _1m_sats: T, + pub _10m_sats: T, + pub _1btc: T, + pub _10btc: T, + pub _100btc: T, + pub _1k_btc: T, + pub _10k_btc: T, +} + +impl ByGreatEqualAmount { + pub fn as_mut_vec(&mut self) -> [&mut T; 13] { + [ + &mut self._1sat, + &mut self._10sats, + &mut self._100sats, + &mut self._1k_sats, + &mut self._10k_sats, + &mut self._100k_sats, + &mut self._1m_sats, + &mut self._10m_sats, + &mut self._1btc, + &mut self._10btc, + &mut self._100btc, + &mut self._1k_btc, + &mut self._10k_btc, + ] + } +} + +impl ByGreatEqualAmount<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 13] { + [ + &self._1sat.1, + &self._10sats.1, + &self._100sats.1, + &self._1k_sats.1, + &self._10k_sats.1, + &self._100k_sats.1, + &self._1m_sats.1, + &self._10m_sats.1, + &self._1btc.1, + &self._10btc.1, + &self._100btc.1, + &self._1k_btc.1, + &self._10k_btc.1, + ] + } +} + +impl From> for ByGreatEqualAmount<(GroupFilter, T)> { + fn from(value: ByGreatEqualAmount) -> Self { + Self { + _1sat: (GroupFilter::GreaterOrEqual(Sats::_1.into()), value._1sat), + _10sats: (GroupFilter::GreaterOrEqual(Sats::_10.into()), value._10sats), + _100sats: ( + GroupFilter::GreaterOrEqual(Sats::_100.into()), + value._100sats, + ), + _1k_sats: ( + GroupFilter::GreaterOrEqual(Sats::_1K.into()), + value._1k_sats, + ), + _10k_sats: ( + GroupFilter::GreaterOrEqual(Sats::_10K.into()), + value._10k_sats, + ), + _100k_sats: ( + GroupFilter::GreaterOrEqual(Sats::_100K.into()), + value._100k_sats, + ), + _1m_sats: ( + GroupFilter::GreaterOrEqual(Sats::_1M.into()), + value._1m_sats, + ), + _10m_sats: ( + GroupFilter::GreaterOrEqual(Sats::_10M.into()), + value._10m_sats, + ), + _1btc: (GroupFilter::GreaterOrEqual(Sats::_1BTC.into()), value._1btc), + _10btc: ( + GroupFilter::GreaterOrEqual(Sats::_10BTC.into()), + value._10btc, + ), + _100btc: ( + GroupFilter::GreaterOrEqual(Sats::_100BTC.into()), + value._100btc, + ), + _1k_btc: ( + GroupFilter::GreaterOrEqual(Sats::_1K_BTC.into()), + value._1k_btc, + ), + _10k_btc: ( + GroupFilter::GreaterOrEqual(Sats::_10K_BTC.into()), + value._10k_btc, + ), + } + } +} diff --git a/crates/brk_core/src/groups/by_lt_amount.rs b/crates/brk_core/src/groups/by_lt_amount.rs new file mode 100644 index 000000000..8fc2191fe --- /dev/null +++ b/crates/brk_core/src/groups/by_lt_amount.rs @@ -0,0 +1,86 @@ +use crate::Sats; + +use super::GroupFilter; + +#[derive(Default, Clone)] +pub struct ByLowerThanAmount { + pub _10sats: T, + pub _100sats: T, + pub _1k_sats: T, + pub _10k_sats: T, + pub _100k_sats: T, + pub _1m_sats: T, + pub _10m_sats: T, + pub _1btc: T, + pub _10btc: T, + pub _100btc: T, + pub _1k_btc: T, + pub _10k_btc: T, + pub _100k_btc: T, +} + +impl ByLowerThanAmount { + pub fn as_mut_vec(&mut self) -> [&mut T; 13] { + [ + &mut self._10sats, + &mut self._100sats, + &mut self._1k_sats, + &mut self._10k_sats, + &mut self._100k_sats, + &mut self._1m_sats, + &mut self._10m_sats, + &mut self._1btc, + &mut self._10btc, + &mut self._100btc, + &mut self._1k_btc, + &mut self._10k_btc, + &mut self._100k_btc, + ] + } +} + +impl ByLowerThanAmount<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 13] { + [ + &self._10sats.1, + &self._100sats.1, + &self._1k_sats.1, + &self._10k_sats.1, + &self._100k_sats.1, + &self._1m_sats.1, + &self._10m_sats.1, + &self._1btc.1, + &self._10btc.1, + &self._100btc.1, + &self._1k_btc.1, + &self._10k_btc.1, + &self._100k_btc.1, + ] + } +} + +impl From> for ByLowerThanAmount<(GroupFilter, T)> { + fn from(value: ByLowerThanAmount) -> Self { + Self { + _10sats: (GroupFilter::LowerThan(Sats::_10.into()), value._10sats), + _100sats: (GroupFilter::LowerThan(Sats::_100.into()), value._100sats), + _1k_sats: (GroupFilter::LowerThan(Sats::_1K.into()), value._1k_sats), + _10k_sats: (GroupFilter::LowerThan(Sats::_10K.into()), value._10k_sats), + _100k_sats: (GroupFilter::LowerThan(Sats::_100K.into()), value._100k_sats), + _1m_sats: (GroupFilter::LowerThan(Sats::_1M.into()), value._1m_sats), + _10m_sats: (GroupFilter::LowerThan(Sats::_10M.into()), value._10m_sats), + _1btc: (GroupFilter::LowerThan(Sats::_1BTC.into()), value._1btc), + _10btc: (GroupFilter::LowerThan(Sats::_10BTC.into()), value._10btc), + _100btc: (GroupFilter::LowerThan(Sats::_100BTC.into()), value._100btc), + _1k_btc: (GroupFilter::LowerThan(Sats::_1K_BTC.into()), value._1k_btc), + _10k_btc: ( + GroupFilter::LowerThan(Sats::_10K_BTC.into()), + value._10k_btc, + ), + _100k_btc: ( + GroupFilter::LowerThan(Sats::_100K_BTC.into()), + value._100k_btc, + ), + } + } +} diff --git a/crates/brk_core/src/groups/by_max_age.rs b/crates/brk_core/src/groups/by_max_age.rs new file mode 100644 index 000000000..225844fc7 --- /dev/null +++ b/crates/brk_core/src/groups/by_max_age.rs @@ -0,0 +1,98 @@ +use super::GroupFilter; + +#[derive(Default, Clone)] +pub struct ByMaxAge { + pub _1w: T, + pub _1m: T, + pub _2m: T, + pub _3m: T, + pub _4m: T, + pub _5m: T, + pub _6m: T, + pub _1y: T, + pub _2y: T, + pub _3y: T, + pub _4y: T, + pub _5y: T, + pub _6y: T, + pub _7y: T, + pub _8y: T, + pub _10y: T, + pub _12y: T, + pub _15y: T, +} + +impl ByMaxAge { + pub fn as_mut_vec(&mut self) -> [&mut T; 18] { + [ + &mut self._1w, + &mut self._1m, + &mut self._2m, + &mut self._3m, + &mut self._4m, + &mut self._5m, + &mut self._6m, + &mut self._1y, + &mut self._2y, + &mut self._3y, + &mut self._4y, + &mut self._5y, + &mut self._6y, + &mut self._7y, + &mut self._8y, + &mut self._10y, + &mut self._12y, + &mut self._15y, + ] + } +} + +impl ByMaxAge<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 18] { + [ + &self._1w.1, + &self._1m.1, + &self._2m.1, + &self._3m.1, + &self._4m.1, + &self._5m.1, + &self._6m.1, + &self._1y.1, + &self._2y.1, + &self._3y.1, + &self._4y.1, + &self._5y.1, + &self._6y.1, + &self._7y.1, + &self._8y.1, + &self._10y.1, + &self._12y.1, + &self._15y.1, + ] + } +} + +impl From> for ByMaxAge<(GroupFilter, T)> { + fn from(value: ByMaxAge) -> Self { + Self { + _1w: (GroupFilter::LowerThan(7), value._1w), + _1m: (GroupFilter::LowerThan(30), value._1m), + _2m: (GroupFilter::LowerThan(2 * 30), value._2m), + _3m: (GroupFilter::LowerThan(3 * 30), value._3m), + _4m: (GroupFilter::LowerThan(4 * 30), value._4m), + _5m: (GroupFilter::LowerThan(5 * 30), value._5m), + _6m: (GroupFilter::LowerThan(6 * 30), value._6m), + _1y: (GroupFilter::LowerThan(365), value._1y), + _2y: (GroupFilter::LowerThan(2 * 365), value._2y), + _3y: (GroupFilter::LowerThan(3 * 365), value._3y), + _4y: (GroupFilter::LowerThan(4 * 365), value._4y), + _5y: (GroupFilter::LowerThan(5 * 365), value._5y), + _6y: (GroupFilter::LowerThan(6 * 365), value._6y), + _7y: (GroupFilter::LowerThan(7 * 365), value._7y), + _8y: (GroupFilter::LowerThan(8 * 365), value._8y), + _10y: (GroupFilter::LowerThan(10 * 365), value._10y), + _12y: (GroupFilter::LowerThan(12 * 365), value._12y), + _15y: (GroupFilter::LowerThan(15 * 365), value._15y), + } + } +} diff --git a/crates/brk_core/src/groups/by_min_age.rs b/crates/brk_core/src/groups/by_min_age.rs new file mode 100644 index 000000000..6a02fb96f --- /dev/null +++ b/crates/brk_core/src/groups/by_min_age.rs @@ -0,0 +1,98 @@ +use super::GroupFilter; + +#[derive(Default, Clone)] +pub struct ByMinAge { + pub _1d: T, + pub _1w: T, + pub _1m: T, + pub _2m: T, + pub _3m: T, + pub _4m: T, + pub _5m: T, + pub _6m: T, + pub _1y: T, + pub _2y: T, + pub _3y: T, + pub _4y: T, + pub _5y: T, + pub _6y: T, + pub _7y: T, + pub _8y: T, + pub _10y: T, + pub _12y: T, +} + +impl ByMinAge { + pub fn as_mut_vec(&mut self) -> [&mut T; 18] { + [ + &mut self._1d, + &mut self._1w, + &mut self._1m, + &mut self._2m, + &mut self._3m, + &mut self._4m, + &mut self._5m, + &mut self._6m, + &mut self._1y, + &mut self._2y, + &mut self._3y, + &mut self._4y, + &mut self._5y, + &mut self._6y, + &mut self._7y, + &mut self._8y, + &mut self._10y, + &mut self._12y, + ] + } +} + +impl ByMinAge<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 18] { + [ + &self._1d.1, + &self._1w.1, + &self._1m.1, + &self._2m.1, + &self._3m.1, + &self._4m.1, + &self._5m.1, + &self._6m.1, + &self._1y.1, + &self._2y.1, + &self._3y.1, + &self._4y.1, + &self._5y.1, + &self._6y.1, + &self._7y.1, + &self._8y.1, + &self._10y.1, + &self._12y.1, + ] + } +} + +impl From> for ByMinAge<(GroupFilter, T)> { + fn from(value: ByMinAge) -> Self { + Self { + _1d: (GroupFilter::GreaterOrEqual(1), value._1d), + _1w: (GroupFilter::GreaterOrEqual(7), value._1w), + _1m: (GroupFilter::GreaterOrEqual(30), value._1m), + _2m: (GroupFilter::GreaterOrEqual(2 * 30), value._2m), + _3m: (GroupFilter::GreaterOrEqual(3 * 30), value._3m), + _4m: (GroupFilter::GreaterOrEqual(4 * 30), value._4m), + _5m: (GroupFilter::GreaterOrEqual(5 * 30), value._5m), + _6m: (GroupFilter::GreaterOrEqual(6 * 30), value._6m), + _1y: (GroupFilter::GreaterOrEqual(365), value._1y), + _2y: (GroupFilter::GreaterOrEqual(2 * 365), value._2y), + _3y: (GroupFilter::GreaterOrEqual(3 * 365), value._3y), + _4y: (GroupFilter::GreaterOrEqual(4 * 365), value._4y), + _5y: (GroupFilter::GreaterOrEqual(5 * 365), value._5y), + _6y: (GroupFilter::GreaterOrEqual(6 * 365), value._6y), + _7y: (GroupFilter::GreaterOrEqual(7 * 365), value._7y), + _8y: (GroupFilter::GreaterOrEqual(8 * 365), value._8y), + _10y: (GroupFilter::GreaterOrEqual(10 * 365), value._10y), + _12y: (GroupFilter::GreaterOrEqual(12 * 365), value._12y), + } + } +} diff --git a/crates/brk_state/src/outputs/by_spendable_type.rs b/crates/brk_core/src/groups/by_spendable_type.rs similarity index 59% rename from crates/brk_state/src/outputs/by_spendable_type.rs rename to crates/brk_core/src/groups/by_spendable_type.rs index 9f404a103..c97042b19 100644 --- a/crates/brk_state/src/outputs/by_spendable_type.rs +++ b/crates/brk_core/src/groups/by_spendable_type.rs @@ -1,11 +1,11 @@ use std::ops::{Add, AddAssign}; -use brk_core::OutputType; +use crate::OutputType; -use super::OutputFilter; +use super::GroupFilter; #[derive(Default, Clone, Debug)] -pub struct OutputsBySpendableType { +pub struct BySpendableType { pub p2pk65: T, pub p2pk33: T, pub p2pkh: T, @@ -19,24 +19,7 @@ pub struct OutputsBySpendableType { pub empty: T, } -impl OutputsBySpendableType { - // pub fn get(&self, output_type: OutputType) -> &T { - // match output_type { - // OutputType::P2PK65 => &self.p2pk65, - // OutputType::P2PK33 => &self.p2pk33, - // OutputType::P2PKH => &self.p2pkh, - // OutputType::P2MS => &self.p2ms, - // OutputType::P2SH => &self.p2sh, - // OutputType::P2WPKH => &self.p2wpkh, - // OutputType::P2WSH => &self.p2wsh, - // OutputType::P2TR => &self.p2tr, - // OutputType::P2A => &self.p2a, - // OutputType::Unknown => &self.unknown, - // OutputType::Empty => &self.empty, - // _ => unreachable!(), - // } - // } - +impl BySpendableType { pub fn get_mut(&mut self, output_type: OutputType) -> &mut T { match output_type { OutputType::P2PK65 => &mut self.p2pk65, @@ -54,22 +37,6 @@ impl OutputsBySpendableType { } } - // pub fn as_vec(&self) -> [&T; 11] { - // [ - // &self.p2pk65, - // &self.p2pk33, - // &self.p2pkh, - // &self.p2ms, - // &self.p2sh, - // &self.p2wpkh, - // &self.p2wsh, - // &self.p2tr, - // &self.p2a, - // &self.unknown, - // &self.empty, - // ] - // } - pub fn as_mut_vec(&mut self) -> [&mut T; 11] { [ &mut self.p2pk65, @@ -103,7 +70,7 @@ impl OutputsBySpendableType { } } -impl OutputsBySpendableType<(OutputFilter, T)> { +impl BySpendableType<(GroupFilter, T)> { pub fn vecs(&self) -> [&T; 11] { [ &self.p2pk65.1, @@ -121,25 +88,25 @@ impl OutputsBySpendableType<(OutputFilter, T)> { } } -impl From> for OutputsBySpendableType<(OutputFilter, T)> { - fn from(value: OutputsBySpendableType) -> Self { +impl From> for BySpendableType<(GroupFilter, T)> { + fn from(value: BySpendableType) -> Self { Self { - p2pk65: (OutputFilter::Type(OutputType::P2PK65), value.p2pk65), - p2pk33: (OutputFilter::Type(OutputType::P2PK33), value.p2pk33), - p2pkh: (OutputFilter::Type(OutputType::P2PKH), value.p2pkh), - p2ms: (OutputFilter::Type(OutputType::P2MS), value.p2ms), - p2sh: (OutputFilter::Type(OutputType::P2SH), value.p2sh), - p2wpkh: (OutputFilter::Type(OutputType::P2WPKH), value.p2wpkh), - p2wsh: (OutputFilter::Type(OutputType::P2WSH), value.p2wsh), - p2tr: (OutputFilter::Type(OutputType::P2TR), value.p2tr), - p2a: (OutputFilter::Type(OutputType::P2A), value.p2a), - unknown: (OutputFilter::Type(OutputType::Unknown), value.unknown), - empty: (OutputFilter::Type(OutputType::Empty), value.empty), + p2pk65: (GroupFilter::Type(OutputType::P2PK65), value.p2pk65), + p2pk33: (GroupFilter::Type(OutputType::P2PK33), value.p2pk33), + p2pkh: (GroupFilter::Type(OutputType::P2PKH), value.p2pkh), + p2ms: (GroupFilter::Type(OutputType::P2MS), value.p2ms), + p2sh: (GroupFilter::Type(OutputType::P2SH), value.p2sh), + p2wpkh: (GroupFilter::Type(OutputType::P2WPKH), value.p2wpkh), + p2wsh: (GroupFilter::Type(OutputType::P2WSH), value.p2wsh), + p2tr: (GroupFilter::Type(OutputType::P2TR), value.p2tr), + p2a: (GroupFilter::Type(OutputType::P2A), value.p2a), + unknown: (GroupFilter::Type(OutputType::Unknown), value.unknown), + empty: (GroupFilter::Type(OutputType::Empty), value.empty), } } } -impl Add for OutputsBySpendableType +impl Add for BySpendableType where T: Add, { @@ -161,7 +128,7 @@ where } } -impl AddAssign for OutputsBySpendableType +impl AddAssign for BySpendableType where T: AddAssign, { diff --git a/crates/brk_core/src/groups/by_term.rs b/crates/brk_core/src/groups/by_term.rs new file mode 100644 index 000000000..9e3978f0b --- /dev/null +++ b/crates/brk_core/src/groups/by_term.rs @@ -0,0 +1,28 @@ +use super::GroupFilter; + +#[derive(Default, Clone)] +pub struct ByTerm { + pub short: T, + pub long: T, +} + +impl ByTerm { + pub fn as_mut_vec(&mut self) -> [&mut T; 2] { + [&mut self.short, &mut self.long] + } +} + +impl ByTerm<(GroupFilter, T)> { + pub fn vecs(&self) -> [&T; 2] { + [&self.short.1, &self.long.1] + } +} + +impl From> for ByTerm<(GroupFilter, T)> { + fn from(value: ByTerm) -> Self { + Self { + short: (GroupFilter::LowerThan(5 * 30), value.short), + long: (GroupFilter::GreaterOrEqual(5 * 30), value.long), + } + } +} diff --git a/crates/brk_state/src/outputs/by_type.rs b/crates/brk_core/src/groups/by_type.rs similarity index 73% rename from crates/brk_state/src/outputs/by_type.rs rename to crates/brk_core/src/groups/by_type.rs index cf2a542bf..dd07592d6 100644 --- a/crates/brk_state/src/outputs/by_type.rs +++ b/crates/brk_core/src/groups/by_type.rs @@ -1,16 +1,16 @@ use std::ops::{Add, AddAssign}; -use brk_core::OutputType; +use crate::OutputType; -use super::{OutputsBySpendableType, OutputsByUnspendableType}; +use super::{BySpendableType, ByUnspendableType}; #[derive(Default, Clone, Debug)] -pub struct OutputsByType { - pub spendable: OutputsBySpendableType, - pub unspendable: OutputsByUnspendableType, +pub struct GroupedByType { + pub spendable: BySpendableType, + pub unspendable: ByUnspendableType, } -impl OutputsByType { +impl GroupedByType { pub fn get(&self, output_type: OutputType) -> &T { match output_type { OutputType::P2PK65 => &self.spendable.p2pk65, @@ -44,25 +44,9 @@ impl OutputsByType { OutputType::OpReturn => &mut self.unspendable.opreturn, } } - - // pub fn as_vec(&self) -> Vec<&T> { - // self.spendable - // .as_vec() - // .into_iter() - // .chain(self.unspendable.as_vec()) - // .collect::>() - // } - - // pub fn as_mut_vec(&mut self) -> Vec<&mut T> { - // self.spendable - // .as_mut_vec() - // .into_iter() - // .chain(self.unspendable.as_mut_vec()) - // .collect::>() - // } } -impl Add for OutputsByType +impl Add for GroupedByType where T: Add, { @@ -75,7 +59,7 @@ where } } -impl AddAssign for OutputsByType +impl AddAssign for GroupedByType where T: AddAssign, { diff --git a/crates/brk_state/src/outputs/by_unspendable_type.rs b/crates/brk_core/src/groups/by_unspendable_type.rs similarity index 72% rename from crates/brk_state/src/outputs/by_unspendable_type.rs rename to crates/brk_core/src/groups/by_unspendable_type.rs index f777a094c..2489a7e7f 100644 --- a/crates/brk_state/src/outputs/by_unspendable_type.rs +++ b/crates/brk_core/src/groups/by_unspendable_type.rs @@ -1,17 +1,17 @@ use std::ops::{Add, AddAssign}; #[derive(Default, Clone, Debug)] -pub struct OutputsByUnspendableType { +pub struct ByUnspendableType { pub opreturn: T, } -impl OutputsByUnspendableType { +impl ByUnspendableType { pub fn as_vec(&self) -> [&T; 1] { [&self.opreturn] } } -impl Add for OutputsByUnspendableType +impl Add for ByUnspendableType where T: Add, { @@ -23,7 +23,7 @@ where } } -impl AddAssign for OutputsByUnspendableType +impl AddAssign for ByUnspendableType where T: AddAssign, { diff --git a/crates/brk_state/src/outputs/by_value.rs b/crates/brk_core/src/groups/by_value.rs similarity index 95% rename from crates/brk_state/src/outputs/by_value.rs rename to crates/brk_core/src/groups/by_value.rs index 8aa4e90fb..f58d311b3 100644 --- a/crates/brk_state/src/outputs/by_value.rs +++ b/crates/brk_core/src/groups/by_value.rs @@ -1,5 +1,5 @@ #[derive(Default, Clone)] -pub struct OutputsByValue { +pub struct GroupedByValue { pub up_to_1cent: T, pub from_1c_to_10c: T, pub from_10c_to_1d: T, @@ -16,7 +16,7 @@ pub struct OutputsByValue { // ... } -impl OutputsByValue { +impl GroupedByValue { pub fn as_mut_vec(&mut self) -> Vec<&mut T> { vec![ &mut self.up_to_1cent, diff --git a/crates/brk_core/src/groups/filter.rs b/crates/brk_core/src/groups/filter.rs new file mode 100644 index 000000000..d6a738b4e --- /dev/null +++ b/crates/brk_core/src/groups/filter.rs @@ -0,0 +1,51 @@ +use std::ops::Range; + +use crate::{HalvingEpoch, OutputType}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GroupFilter { + All, + LowerThan(usize), + Range(Range), + GreaterOrEqual(usize), + Epoch(HalvingEpoch), + Type(OutputType), +} + +impl GroupFilter { + pub fn contains(&self, value: usize) -> bool { + match self { + GroupFilter::All => true, + GroupFilter::LowerThan(max) => *max > value, + GroupFilter::GreaterOrEqual(min) => *min <= value, + GroupFilter::Range(r) => r.contains(&value), + GroupFilter::Epoch(_) => false, + GroupFilter::Type(_) => false, + } + } + + pub fn includes(&self, other: &GroupFilter) -> bool { + match self { + GroupFilter::All => true, + GroupFilter::LowerThan(max) => match other { + GroupFilter::All => false, + GroupFilter::LowerThan(max2) => max >= max2, + GroupFilter::Range(range) => range.end <= *max, + GroupFilter::GreaterOrEqual(_) => false, + GroupFilter::Epoch(_) => false, + GroupFilter::Type(_) => false, + }, + GroupFilter::GreaterOrEqual(min) => match other { + GroupFilter::All => false, + GroupFilter::LowerThan(_) => false, + GroupFilter::Range(range) => range.start >= *min, + GroupFilter::GreaterOrEqual(min2) => min <= min2, + GroupFilter::Epoch(_) => false, + GroupFilter::Type(_) => false, + }, + GroupFilter::Range(_) => false, + GroupFilter::Epoch(_) => false, + GroupFilter::Type(_) => false, + } + } +} diff --git a/crates/brk_core/src/groups/mod.rs b/crates/brk_core/src/groups/mod.rs new file mode 100644 index 000000000..8ca92df9f --- /dev/null +++ b/crates/brk_core/src/groups/mod.rs @@ -0,0 +1,31 @@ +mod address; +mod by_address_type; +mod by_age_range; +mod by_amount_range; +mod by_epoch; +mod by_ge_amount; +mod by_lt_amount; +mod by_max_age; +mod by_min_age; +mod by_spendable_type; +mod by_term; +mod by_type; +mod by_unspendable_type; +mod filter; +mod utxo; + +pub use address::*; +pub use by_address_type::*; +pub use by_age_range::*; +pub use by_amount_range::*; +pub use by_epoch::*; +pub use by_ge_amount::*; +pub use by_lt_amount::*; +pub use by_max_age::*; +pub use by_min_age::*; +pub use by_spendable_type::*; +pub use by_term::*; +pub use by_type::*; +pub use by_unspendable_type::*; +pub use filter::*; +pub use utxo::*; diff --git a/crates/brk_core/src/groups/utxo.rs b/crates/brk_core/src/groups/utxo.rs new file mode 100644 index 000000000..05bc77921 --- /dev/null +++ b/crates/brk_core/src/groups/utxo.rs @@ -0,0 +1,90 @@ +use crate::{ + ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, ByMaxAge, ByMinAge, + BySpendableType, ByTerm, GroupFilter, +}; + +#[derive(Default, Clone)] +pub struct UTXOGroups { + pub all: T, + pub age_range: ByAgeRange, + pub epoch: ByEpoch, + pub min_age: ByMinAge, + pub ge_amount: ByGreatEqualAmount, + pub amount_range: ByAmountRange, + pub term: ByTerm, + pub _type: BySpendableType, + pub max_age: ByMaxAge, + pub lt_amount: ByLowerThanAmount, +} + +impl UTXOGroups { + pub fn as_mut_vecs(&mut self) -> Vec<&mut T> { + [&mut self.all] + .into_iter() + .chain(self.term.as_mut_vec()) + .chain(self.max_age.as_mut_vec()) + .chain(self.min_age.as_mut_vec()) + .chain(self.ge_amount.as_mut_vec()) + .chain(self.age_range.as_mut_vec()) + .chain(self.epoch.as_mut_vec()) + .chain(self.amount_range.as_mut_vec()) + .chain(self.lt_amount.as_mut_vec()) + .chain(self._type.as_mut_vec()) + .collect::>() + } + + pub fn as_mut_separate_vecs(&mut self) -> Vec<&mut T> { + self.age_range + .as_mut_vec() + .into_iter() + .chain(self.epoch.as_mut_vec()) + .chain(self.amount_range.as_mut_vec()) + .chain(self._type.as_mut_vec()) + .collect::>() + } + + pub fn as_mut_overlapping_vecs(&mut self) -> Vec<&mut T> { + [&mut self.all] + .into_iter() + .chain(self.term.as_mut_vec()) + .chain(self.max_age.as_mut_vec()) + .chain(self.min_age.as_mut_vec()) + .chain(self.lt_amount.as_mut_vec()) + .chain(self.ge_amount.as_mut_vec()) + .collect::>() + } +} + +impl UTXOGroups<(GroupFilter, T)> { + pub fn vecs(&self) -> Vec<&T> { + [&self.all.1] + .into_iter() + .chain(self.term.vecs()) + .chain(self.max_age.vecs()) + .chain(self.min_age.vecs()) + .chain(self.age_range.vecs()) + .chain(self.epoch.vecs()) + .chain(self.amount_range.vecs()) + .chain(self._type.vecs()) + .chain(self.lt_amount.vecs()) + .chain(self.ge_amount.vecs()) + .collect::>() + } +} + +impl From> for UTXOGroups<(GroupFilter, T)> { + fn from(value: UTXOGroups) -> Self { + Self { + all: (GroupFilter::All, value.all), + term: ByTerm::from(value.term), + max_age: ByMaxAge::from(value.max_age), + min_age: ByMinAge::from(value.min_age), + age_range: ByAgeRange::from(value.age_range), + epoch: ByEpoch::from(value.epoch), + amount_range: ByAmountRange::from(value.amount_range), + lt_amount: ByLowerThanAmount::from(value.lt_amount), + ge_amount: ByGreatEqualAmount::from(value.ge_amount), + _type: BySpendableType::from(value._type), + } + } +} diff --git a/crates/brk_core/src/lib.rs b/crates/brk_core/src/lib.rs index 6e3598d9f..ffffbbe39 100644 --- a/crates/brk_core/src/lib.rs +++ b/crates/brk_core/src/lib.rs @@ -1,11 +1,13 @@ #![doc = include_str!("../README.md")] -mod enums; +mod error; +mod groups; mod structs; mod traits; mod utils; -pub use enums::*; +pub use error::*; +pub use groups::*; pub use structs::*; pub use traits::*; pub use utils::*; diff --git a/crates/brk_core/src/structs/addressbytes.rs b/crates/brk_core/src/structs/addressbytes.rs index d22cf485f..cc4125608 100644 --- a/crates/brk_core/src/structs/addressbytes.rs +++ b/crates/brk_core/src/structs/addressbytes.rs @@ -162,7 +162,7 @@ impl fmt::Display for P2PKHBytes { .push_opcode(opcodes::all::OP_CHECKSIG) .into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -192,7 +192,7 @@ impl fmt::Display for P2SHBytes { .push_opcode(opcodes::all::OP_EQUAL) .into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -218,7 +218,7 @@ impl fmt::Display for P2WPKHBytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(0).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -244,7 +244,7 @@ impl fmt::Display for P2WSHBytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(0).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -270,7 +270,7 @@ impl fmt::Display for P2TRBytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(1).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } @@ -296,7 +296,7 @@ impl fmt::Display for P2ABytes { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let script = Builder::new().push_int(1).push_slice(*self.0).into_script(); let address = Address::from_script(&script, Network::Bitcoin).unwrap(); - write!(f, "{}", address) + write!(f, "{address}") } } diff --git a/crates/brk_core/src/structs/addressdata.rs b/crates/brk_core/src/structs/addressdata.rs new file mode 100644 index 000000000..c80148ee6 --- /dev/null +++ b/crates/brk_core/src/structs/addressdata.rs @@ -0,0 +1,102 @@ +use byteview::ByteView; +use zerocopy::{FromBytes, IntoBytes}; + +use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats}; + +#[derive(Debug, Default, Clone)] +#[repr(C)] +pub struct AddressData { + pub sent: Sats, + pub received: Sats, + pub realized_cap: Dollars, + pub outputs_len: u32, +} + +impl AddressData { + pub fn amount(&self) -> Sats { + (u64::from(self.received) - u64::from(self.sent)).into() + } + + pub fn realized_price(&self) -> Dollars { + (self.realized_cap / Bitcoin::from(self.amount())).round_to_4_digits() + } + + #[inline] + pub fn has_0_sats(&self) -> bool { + self.amount() == Sats::ZERO + } + + #[inline] + pub fn has_0_utxos(&self) -> bool { + self.outputs_len == 0 + } + + pub fn receive(&mut self, amount: Sats, price: Option) { + self.received += amount; + self.outputs_len += 1; + if let Some(price) = price { + self.realized_cap += price * amount; + } + } + + pub fn send(&mut self, amount: Sats, previous_price: Option) -> Result<()> { + if self.amount() < amount { + return Err(Error::String("Previous_amount smaller than sent amount")); + } + self.sent += amount; + self.outputs_len -= 1; + if let Some(previous_price) = previous_price { + self.realized_cap = self + .realized_cap + .checked_sub(previous_price * amount) + .unwrap(); + } + Ok(()) + } +} + +impl From for AddressData { + fn from(value: EmptyAddressData) -> Self { + Self::from(&value) + } +} +impl From<&EmptyAddressData> for AddressData { + fn from(value: &EmptyAddressData) -> Self { + Self { + sent: value.transfered, + received: value.transfered, + realized_cap: Dollars::ZERO, + outputs_len: 0, + } + } +} + +impl From for AddressData { + fn from(value: ByteView) -> Self { + Self { + // MUST be same order as impl From<&AddressData> for ByteView + sent: Sats::read_from_bytes(&value[..8]).unwrap(), + received: Sats::read_from_bytes(&value[8..16]).unwrap(), + realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(), + outputs_len: u32::read_from_bytes(&value[24..]).unwrap(), + } + } +} +impl From for ByteView { + fn from(value: AddressData) -> Self { + Self::from(&value) + } +} +impl From<&AddressData> for ByteView { + fn from(value: &AddressData) -> Self { + Self::new( + &[ + value.sent.as_bytes(), + value.received.as_bytes(), + value.realized_cap.as_bytes(), + value.outputs_len.as_bytes(), + ] + .concat(), + ) + } +} diff --git a/crates/brk_core/src/structs/bitcoin.rs b/crates/brk_core/src/structs/bitcoin.rs index 85f911a74..84257c818 100644 --- a/crates/brk_core/src/structs/bitcoin.rs +++ b/crates/brk_core/src/structs/bitcoin.rs @@ -1,6 +1,6 @@ use std::{ cmp::Ordering, - ops::{Add, Div, Mul}, + ops::{Add, AddAssign, Div, Mul}, }; use serde::Serialize; @@ -20,6 +20,12 @@ impl Add for Bitcoin { } } +impl AddAssign for Bitcoin { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl Mul for Bitcoin { type Output = Self; fn mul(self, rhs: Self) -> Self::Output { diff --git a/crates/brk_core/src/structs/cents.rs b/crates/brk_core/src/structs/cents.rs index 2aa93eca1..3e8750dd9 100644 --- a/crates/brk_core/src/structs/cents.rs +++ b/crates/brk_core/src/structs/cents.rs @@ -24,10 +24,28 @@ use super::Dollars; )] pub struct Cents(i64); +const SIGNIFICANT_DIGITS: i32 = 4; + impl Cents { pub const fn mint(value: i64) -> Self { Self(value) } + + pub fn round_to_4_digits(self) -> Self { + let v = self.0; + + let ilog10 = v.checked_ilog10().unwrap_or(0) as i32; + + Self::from(if ilog10 >= SIGNIFICANT_DIGITS { + let log_diff = ilog10 - SIGNIFICANT_DIGITS + 1; + + let pow = 10.0_f64.powi(log_diff); + + ((v as f64 / pow).round() * pow) as i64 + } else { + v + }) + } } impl From for Cents { diff --git a/crates/brk_core/src/structs/date.rs b/crates/brk_core/src/structs/date.rs index e3ed7398c..f1fe488be 100644 --- a/crates/brk_core/src/structs/date.rs +++ b/crates/brk_core/src/structs/date.rs @@ -31,6 +31,10 @@ impl Date { pub fn day(&self) -> u8 { (self.0 % 1_00) as u8 } + + pub fn into_jiff(self) -> Date_ { + self.into() + } } impl Default for Date { diff --git a/crates/brk_core/src/structs/dateindex.rs b/crates/brk_core/src/structs/dateindex.rs index 7c079ef26..1b47506c2 100644 --- a/crates/brk_core/src/structs/dateindex.rs +++ b/crates/brk_core/src/structs/dateindex.rs @@ -3,11 +3,14 @@ use std::{ ops::{Add, Rem}, }; +use jiff::Span; use serde::Serialize; -// use color_eyre::eyre::eyre; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{CheckedSub, Error, Printable}; +use crate::{ + CheckedSub, DecadeIndex, Error, FromCoarserIndex, MonthIndex, Printable, QuarterIndex, + SemesterIndex, WeekIndex, YearIndex, +}; use super::Date; @@ -103,3 +106,135 @@ impl Printable for DateIndex { &["d", "date", "dateindex"] } } + +impl FromCoarserIndex for DateIndex { + fn min_from(coarser: WeekIndex) -> usize { + let coarser = usize::from(coarser); + if coarser == 0 { + 0 + } else if coarser == 1 { + 1 + } else { + 4 + (coarser - 2) * 7 + } + } + + fn max_from_(coarser: WeekIndex) -> usize { + let coarser = usize::from(coarser); + if coarser == 0 { + 0 + } else if coarser == 1 { + 3 + } else { + 3 + (coarser - 1) * 7 + } + } +} + +impl FromCoarserIndex for DateIndex { + fn min_from(coarser: MonthIndex) -> usize { + let coarser = u16::from(coarser); + if coarser == 0 { + 0 + } else { + let d = Date::new(2009, 1, 1) + .into_jiff() + .checked_add(Span::new().months(coarser)) + .unwrap(); + DateIndex::try_from(Date::from(d)).unwrap().into() + } + } + + fn max_from_(coarser: MonthIndex) -> usize { + let d = Date::new(2009, 1, 31) + .into_jiff() + .checked_add(Span::new().months(u16::from(coarser))) + .unwrap(); + DateIndex::try_from(Date::from(d)).unwrap().into() + } +} + +impl FromCoarserIndex for DateIndex { + fn min_from(coarser: QuarterIndex) -> usize { + let coarser = u16::from(coarser); + if coarser == 0 { + 0 + } else { + let d = Date::new(2009, 1, 1) + .into_jiff() + .checked_add(Span::new().months(3 * coarser)) + .unwrap(); + DateIndex::try_from(Date::from(d)).unwrap().into() + } + } + + fn max_from_(coarser: QuarterIndex) -> usize { + let d = Date::new(2009, 3, 31) + .into_jiff() + .checked_add(Span::new().months(3 * u16::from(coarser))) + .unwrap(); + DateIndex::try_from(Date::from(d)).unwrap().into() + } +} + +impl FromCoarserIndex for DateIndex { + fn min_from(coarser: SemesterIndex) -> usize { + let coarser = u16::from(coarser); + if coarser == 0 { + 0 + } else { + let d = Date::new(2009, 1, 1) + .into_jiff() + .checked_add(Span::new().months(6 * coarser)) + .unwrap(); + DateIndex::try_from(Date::from(d)).unwrap().into() + } + } + + fn max_from_(coarser: SemesterIndex) -> usize { + let d = Date::new(2009, 5, 31) + .into_jiff() + .checked_add(Span::new().months(1 + 6 * u16::from(coarser))) + .unwrap(); + DateIndex::try_from(Date::from(d)).unwrap().into() + } +} + +impl FromCoarserIndex for DateIndex { + fn min_from(coarser: YearIndex) -> usize { + let coarser = u16::from(coarser); + if coarser == 0 { + 0 + } else { + Self::try_from(Date::new(2009 + coarser, 1, 1)) + .unwrap() + .into() + } + } + + fn max_from_(coarser: YearIndex) -> usize { + Self::try_from(Date::new(2009 + u16::from(coarser), 12, 31)) + .unwrap() + .into() + } +} + +impl FromCoarserIndex for DateIndex { + fn min_from(coarser: DecadeIndex) -> usize { + let coarser = u16::from(coarser); + if coarser == 0 { + 0 + } else { + Self::try_from(Date::new(2000 + 10 * coarser, 1, 1)) + .unwrap() + .into() + } + } + + fn max_from_(coarser: DecadeIndex) -> usize { + let coarser = u16::from(coarser); + Self::try_from(Date::new(2009 + (10 * coarser), 12, 31)) + .unwrap() + .into() + } +} diff --git a/crates/brk_core/src/structs/decadeindex.rs b/crates/brk_core/src/structs/decadeindex.rs index 5b6e806e6..e02a5a7ff 100644 --- a/crates/brk_core/src/structs/decadeindex.rs +++ b/crates/brk_core/src/structs/decadeindex.rs @@ -1,4 +1,7 @@ -use std::{fmt::Debug, ops::Add}; +use std::{ + fmt::Debug, + ops::{Add, AddAssign, Div}, +}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -31,6 +34,18 @@ impl From for DecadeIndex { } } +impl From for u8 { + fn from(value: DecadeIndex) -> Self { + value.0 + } +} + +impl From for u16 { + fn from(value: DecadeIndex) -> Self { + value.0 as u16 + } +} + impl From for DecadeIndex { fn from(value: usize) -> Self { Self(value as u8) @@ -51,6 +66,27 @@ impl Add for DecadeIndex { } } +impl Add for DecadeIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self::from(self.0 + rhs.0) + } +} + +impl AddAssign for DecadeIndex { + fn add_assign(&mut self, rhs: Self) { + *self = Self(self.0 + rhs.0) + } +} + +impl Div for DecadeIndex { + type Output = Self; + fn div(self, _: usize) -> Self::Output { + unreachable!() + } +} + impl From for DecadeIndex { fn from(value: DateIndex) -> Self { Self::from(Date::from(value)) diff --git a/crates/brk_core/src/structs/difficultyepoch.rs b/crates/brk_core/src/structs/difficultyepoch.rs index 3886ab9a6..fa5772a4f 100644 --- a/crates/brk_core/src/structs/difficultyepoch.rs +++ b/crates/brk_core/src/structs/difficultyepoch.rs @@ -1,6 +1,6 @@ use std::{ fmt::Debug, - ops::{Add, Div}, + ops::{Add, AddAssign, Div}, }; use serde::{Deserialize, Serialize}; @@ -54,6 +54,12 @@ impl Add for DifficultyEpoch { } } +impl AddAssign for DifficultyEpoch { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl Add for DifficultyEpoch { type Output = Self; diff --git a/crates/brk_core/src/structs/dollars.rs b/crates/brk_core/src/structs/dollars.rs index 0cbb62303..6b3a20d19 100644 --- a/crates/brk_core/src/structs/dollars.rs +++ b/crates/brk_core/src/structs/dollars.rs @@ -38,6 +38,14 @@ impl Dollars { pub const fn mint(dollars: f64) -> Self { Self(dollars) } + + pub fn round_nearest_cent(self) -> Self { + Dollars((self.0 * 100.0).round() / 100.0) + } + + pub fn round_to_4_digits(self) -> Self { + Self::from(Cents::from(self).round_to_4_digits()) + } } impl From for Dollars { @@ -235,7 +243,7 @@ impl Mul for Dollars { self } else { Self::from(Cents::from( - u128::from(rhs) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC), + u128::from(rhs) * u128::from(Cents::from(self)) / Sats::ONE_BTC_U128, )) } } diff --git a/crates/brk_core/src/structs/emptyaddressdata.rs b/crates/brk_core/src/structs/emptyaddressdata.rs new file mode 100644 index 000000000..d081ed0e8 --- /dev/null +++ b/crates/brk_core/src/structs/emptyaddressdata.rs @@ -0,0 +1,44 @@ +use byteview::ByteView; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::{AddressData, Sats}; + +#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)] +pub struct EmptyAddressData { + pub transfered: Sats, +} + +impl From for EmptyAddressData { + fn from(value: AddressData) -> Self { + Self::from(&value) + } +} + +impl From<&AddressData> for EmptyAddressData { + fn from(value: &AddressData) -> Self { + if value.sent != value.received { + dbg!(&value); + panic!("Trying to convert not empty wallet to empty !"); + } + Self { + transfered: value.sent, + } + } +} + +impl From for EmptyAddressData { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: EmptyAddressData) -> Self { + Self::from(&value) + } +} +impl From<&EmptyAddressData> for ByteView { + fn from(value: &EmptyAddressData) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/feerate.rs b/crates/brk_core/src/structs/feerate.rs index 35b1cd767..d22c10f90 100644 --- a/crates/brk_core/src/structs/feerate.rs +++ b/crates/brk_core/src/structs/feerate.rs @@ -1,6 +1,6 @@ use std::{ cmp::Ordering, - ops::{Add, Div}, + ops::{Add, AddAssign, Div}, }; use serde::Serialize; @@ -35,6 +35,12 @@ impl Add for Feerate { } } +impl AddAssign for Feerate { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl Div for Feerate { type Output = Self; fn div(self, rhs: usize) -> Self::Output { diff --git a/crates/brk_core/src/structs/halvingepoch.rs b/crates/brk_core/src/structs/halvingepoch.rs index 66cf19401..cb2e3866e 100644 --- a/crates/brk_core/src/structs/halvingepoch.rs +++ b/crates/brk_core/src/structs/halvingepoch.rs @@ -1,6 +1,6 @@ use std::{ fmt::Debug, - ops::{Add, Div}, + ops::{Add, AddAssign, Div}, }; use serde::{Deserialize, Serialize}; @@ -60,6 +60,12 @@ impl Add for HalvingEpoch { } } +impl AddAssign for HalvingEpoch { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl Add for HalvingEpoch { type Output = Self; diff --git a/crates/brk_core/src/structs/mod.rs b/crates/brk_core/src/structs/mod.rs index 32142545b..e21c12922 100644 --- a/crates/brk_core/src/structs/mod.rs +++ b/crates/brk_core/src/structs/mod.rs @@ -1,5 +1,6 @@ mod addressbytes; mod addressbyteshash; +mod addressdata; mod bitcoin; mod blockhash; mod blockhashprefix; @@ -9,6 +10,7 @@ mod dateindex; mod decadeindex; mod difficultyepoch; mod dollars; +mod emptyaddressdata; mod emptyoutputindex; mod feerate; mod halvingepoch; @@ -31,6 +33,7 @@ mod p2wshaddressindex; mod quarterindex; mod rawlocktime; mod sats; +mod semesterindex; mod stored_f32; mod stored_f64; mod stored_u32; @@ -43,6 +46,7 @@ mod txidprefix; mod txindex; mod txversion; mod typeindex; +mod typeindex_with_outputindex; mod unit; mod unknownoutputindex; mod version; @@ -54,6 +58,7 @@ mod yearindex; pub use addressbytes::*; pub use addressbyteshash::*; +pub use addressdata::*; pub use bitcoin::*; pub use blockhash::*; pub use blockhashprefix::*; @@ -63,6 +68,7 @@ pub use dateindex::*; pub use decadeindex::*; pub use difficultyepoch::*; pub use dollars::*; +pub use emptyaddressdata::*; pub use emptyoutputindex::*; pub use feerate::*; pub use halvingepoch::*; @@ -85,6 +91,7 @@ pub use p2wshaddressindex::*; pub use quarterindex::*; pub use rawlocktime::*; pub use sats::*; +pub use semesterindex::*; pub use stored_f32::*; pub use stored_f64::*; pub use stored_u8::*; @@ -97,6 +104,7 @@ pub use txidprefix::*; pub use txindex::*; pub use txversion::*; pub use typeindex::*; +pub use typeindex_with_outputindex::*; pub use unit::*; pub use unknownoutputindex::*; pub use version::*; diff --git a/crates/brk_core/src/structs/monthindex.rs b/crates/brk_core/src/structs/monthindex.rs index d99f95b72..94fdfb4ea 100644 --- a/crates/brk_core/src/structs/monthindex.rs +++ b/crates/brk_core/src/structs/monthindex.rs @@ -1,4 +1,7 @@ -use std::{fmt::Debug, ops::Add}; +use std::{ + fmt::Debug, + ops::{Add, AddAssign, Div}, +}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -31,6 +34,12 @@ impl From for MonthIndex { } } +impl From for u16 { + fn from(value: MonthIndex) -> Self { + value.0 + } +} + impl From for MonthIndex { fn from(value: usize) -> Self { Self(value as u16) @@ -51,6 +60,27 @@ impl Add for MonthIndex { } } +impl Add for MonthIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self::from(self.0 + rhs.0) + } +} + +impl AddAssign for MonthIndex { + fn add_assign(&mut self, rhs: Self) { + *self = Self(self.0 + rhs.0) + } +} + +impl Div for MonthIndex { + type Output = Self; + fn div(self, _: usize) -> Self::Output { + unreachable!() + } +} + impl From for MonthIndex { fn from(value: DateIndex) -> Self { Self::from(Date::from(value)) diff --git a/crates/brk_core/src/structs/ohlc.rs b/crates/brk_core/src/structs/ohlc.rs index 7afaceda2..4e13773b4 100644 --- a/crates/brk_core/src/structs/ohlc.rs +++ b/crates/brk_core/src/structs/ohlc.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, AddAssign, Div}; use derive_deref::{Deref, DerefMut}; use serde::{Serialize, Serializer, ser::SerializeTuple}; @@ -236,6 +236,15 @@ where } } +impl AddAssign for Open +where + T: Add + Clone, +{ + fn add_assign(&mut self, rhs: Self) { + **self = self.0.clone() + rhs.0 + } +} + impl Div for Open where T: Div, @@ -324,6 +333,15 @@ where } } +impl AddAssign for High +where + T: Add + Clone, +{ + fn add_assign(&mut self, rhs: Self) { + **self = self.0.clone() + rhs.0 + } +} + impl Div for High where T: Div, @@ -412,6 +430,15 @@ where } } +impl AddAssign for Low +where + T: Add + Clone, +{ + fn add_assign(&mut self, rhs: Self) { + **self = self.0.clone() + rhs.0 + } +} + impl Div for Low where T: Div, @@ -515,6 +542,15 @@ where } } +impl AddAssign for Close +where + T: Add + Clone, +{ + fn add_assign(&mut self, rhs: Self) { + **self = self.0.clone() + rhs.0 + } +} + impl Div for Close where T: Div, diff --git a/crates/brk_core/src/structs/outputindex.rs b/crates/brk_core/src/structs/outputindex.rs index 6766ce675..80e1e90a9 100644 --- a/crates/brk_core/src/structs/outputindex.rs +++ b/crates/brk_core/src/structs/outputindex.rs @@ -28,6 +28,8 @@ use super::Vout; pub struct OutputIndex(u64); impl OutputIndex { + pub const ZERO: Self = Self(0); + pub const COINBASE: Self = Self(u64::MAX); pub fn incremented(self) -> Self { diff --git a/crates/brk_core/src/structs/outputtype.rs b/crates/brk_core/src/structs/outputtype.rs index 4a4c70c4f..4f1eab597 100644 --- a/crates/brk_core/src/structs/outputtype.rs +++ b/crates/brk_core/src/structs/outputtype.rs @@ -50,6 +50,27 @@ impl OutputType { } } + pub fn is_address(&self) -> bool { + match self { + Self::P2PK65 => true, + Self::P2PK33 => true, + Self::P2PKH => true, + Self::P2MS => false, + Self::P2SH => true, + Self::OpReturn => false, + Self::P2WPKH => true, + Self::P2WSH => true, + Self::P2TR => true, + Self::P2A => true, + Self::Empty => false, + Self::Unknown => false, + } + } + + pub fn is_not_address(&self) -> bool { + !self.is_address() + } + pub fn is_unspendable(&self) -> bool { !self.is_spendable() } diff --git a/crates/brk_core/src/structs/p2aaddressindex.rs b/crates/brk_core/src/structs/p2aaddressindex.rs index 9e268b9ef..6c98a44a3 100644 --- a/crates/brk_core/src/structs/p2aaddressindex.rs +++ b/crates/brk_core/src/structs/p2aaddressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,6 +31,21 @@ impl From for P2AAddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2AAddressIndex) -> Self { + value.0 + } +} +impl From for u32 { + fn from(value: P2AAddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2AAddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for usize { fn from(value: P2AAddressIndex) -> Self { Self::from(*value) @@ -59,3 +76,19 @@ impl Printable for P2AAddressIndex { &["aaddr", "p2aaddr", "p2aaddressindex"] } } + +impl From for P2AAddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2AAddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2AAddressIndex> for ByteView { + fn from(value: &P2AAddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/p2pk33addressindex.rs b/crates/brk_core/src/structs/p2pk33addressindex.rs index e925af645..c6f5aa7de 100644 --- a/crates/brk_core/src/structs/p2pk33addressindex.rs +++ b/crates/brk_core/src/structs/p2pk33addressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,6 +31,21 @@ impl From for P2PK33AddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2PK33AddressIndex) -> Self { + value.0 + } +} +impl From for u32 { + fn from(value: P2PK33AddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2PK33AddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for usize { fn from(value: P2PK33AddressIndex) -> Self { Self::from(*value) @@ -60,3 +77,19 @@ impl Printable for P2PK33AddressIndex { &["pk33addr", "p2pk33addr", "p2pk33addressindex"] } } + +impl From for P2PK33AddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2PK33AddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2PK33AddressIndex> for ByteView { + fn from(value: &P2PK33AddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/p2pk65addressindex.rs b/crates/brk_core/src/structs/p2pk65addressindex.rs index a16b3a3e6..8de0a1df0 100644 --- a/crates/brk_core/src/structs/p2pk65addressindex.rs +++ b/crates/brk_core/src/structs/p2pk65addressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,11 +31,26 @@ impl From for P2PK65AddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2PK65AddressIndex) -> Self { + value.0 + } +} impl From for usize { fn from(value: P2PK65AddressIndex) -> Self { Self::from(*value) } } +impl From for u32 { + fn from(value: P2PK65AddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2PK65AddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for P2PK65AddressIndex { fn from(value: usize) -> Self { Self(TypeIndex::from(value)) @@ -59,3 +76,19 @@ impl Printable for P2PK65AddressIndex { &["pk65addr", "p2pk65addr", "p2pk65addressindex"] } } + +impl From for P2PK65AddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2PK65AddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2PK65AddressIndex> for ByteView { + fn from(value: &P2PK65AddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/p2pkhaddressindex.rs b/crates/brk_core/src/structs/p2pkhaddressindex.rs index ca6e8c729..a481af106 100644 --- a/crates/brk_core/src/structs/p2pkhaddressindex.rs +++ b/crates/brk_core/src/structs/p2pkhaddressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,11 +31,26 @@ impl From for P2PKHAddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2PKHAddressIndex) -> Self { + value.0 + } +} impl From for usize { fn from(value: P2PKHAddressIndex) -> Self { Self::from(*value) } } +impl From for u32 { + fn from(value: P2PKHAddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2PKHAddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for P2PKHAddressIndex { fn from(value: usize) -> Self { Self(TypeIndex::from(value)) @@ -60,3 +77,19 @@ impl Printable for P2PKHAddressIndex { &["pkhaddr", "p2pkhaddr", "p2pkhaddressindex"] } } + +impl From for P2PKHAddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2PKHAddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2PKHAddressIndex> for ByteView { + fn from(value: &P2PKHAddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/p2shaddressindex.rs b/crates/brk_core/src/structs/p2shaddressindex.rs index 90dff9c7a..4ee432881 100644 --- a/crates/brk_core/src/structs/p2shaddressindex.rs +++ b/crates/brk_core/src/structs/p2shaddressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,6 +31,21 @@ impl From for P2SHAddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2SHAddressIndex) -> Self { + value.0 + } +} +impl From for u32 { + fn from(value: P2SHAddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2SHAddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for usize { fn from(value: P2SHAddressIndex) -> Self { Self::from(*value) @@ -60,3 +77,19 @@ impl Printable for P2SHAddressIndex { &["shaddr", "p2shaddr", "p2shaddressindex"] } } + +impl From for P2SHAddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2SHAddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2SHAddressIndex> for ByteView { + fn from(value: &P2SHAddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/p2traddressindex.rs b/crates/brk_core/src/structs/p2traddressindex.rs index ff6633516..7f76cb713 100644 --- a/crates/brk_core/src/structs/p2traddressindex.rs +++ b/crates/brk_core/src/structs/p2traddressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,6 +31,21 @@ impl From for P2TRAddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2TRAddressIndex) -> Self { + value.0 + } +} +impl From for u32 { + fn from(value: P2TRAddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2TRAddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for usize { fn from(value: P2TRAddressIndex) -> Self { Self::from(*value) @@ -60,3 +77,19 @@ impl Printable for P2TRAddressIndex { &["traddr", "p2traddr", "p2traddressindex"] } } + +impl From for P2TRAddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2TRAddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2TRAddressIndex> for ByteView { + fn from(value: &P2TRAddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/p2wpkhaddressindex.rs b/crates/brk_core/src/structs/p2wpkhaddressindex.rs index fb6528b80..dbda26d63 100644 --- a/crates/brk_core/src/structs/p2wpkhaddressindex.rs +++ b/crates/brk_core/src/structs/p2wpkhaddressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,11 +31,26 @@ impl From for P2WPKHAddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2WPKHAddressIndex) -> Self { + value.0 + } +} impl From for usize { fn from(value: P2WPKHAddressIndex) -> Self { Self::from(*value) } } +impl From for u32 { + fn from(value: P2WPKHAddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2WPKHAddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for P2WPKHAddressIndex { fn from(value: usize) -> Self { Self(TypeIndex::from(value)) @@ -60,3 +77,19 @@ impl Printable for P2WPKHAddressIndex { &["wpkhaddr", "p2wpkhaddr", "p2wpkhaddressindex"] } } + +impl From for P2WPKHAddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2WPKHAddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2WPKHAddressIndex> for ByteView { + fn from(value: &P2WPKHAddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/p2wshaddressindex.rs b/crates/brk_core/src/structs/p2wshaddressindex.rs index 6a61485cb..3d92cd598 100644 --- a/crates/brk_core/src/structs/p2wshaddressindex.rs +++ b/crates/brk_core/src/structs/p2wshaddressindex.rs @@ -1,7 +1,9 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Printable, TypeIndex}; @@ -29,6 +31,21 @@ impl From for P2WSHAddressIndex { Self(value) } } +impl From for TypeIndex { + fn from(value: P2WSHAddressIndex) -> Self { + value.0 + } +} +impl From for u32 { + fn from(value: P2WSHAddressIndex) -> Self { + Self::from(*value) + } +} +impl From for P2WSHAddressIndex { + fn from(value: u32) -> Self { + Self(TypeIndex::from(value)) + } +} impl From for usize { fn from(value: P2WSHAddressIndex) -> Self { Self::from(*value) @@ -60,3 +77,19 @@ impl Printable for P2WSHAddressIndex { &["wshaddr", "p2wshaddr", "p2wshaddressindex"] } } + +impl From for P2WSHAddressIndex { + fn from(value: ByteView) -> Self { + Self::read_from_bytes(&value).unwrap() + } +} +impl From for ByteView { + fn from(value: P2WSHAddressIndex) -> Self { + Self::from(&value) + } +} +impl From<&P2WSHAddressIndex> for ByteView { + fn from(value: &P2WSHAddressIndex) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/crates/brk_core/src/structs/quarterindex.rs b/crates/brk_core/src/structs/quarterindex.rs index 6f77bb7a5..3ace803ae 100644 --- a/crates/brk_core/src/structs/quarterindex.rs +++ b/crates/brk_core/src/structs/quarterindex.rs @@ -1,4 +1,7 @@ -use std::{fmt::Debug, ops::Add}; +use std::{ + fmt::Debug, + ops::{Add, AddAssign, Div}, +}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -37,6 +40,12 @@ impl From for QuarterIndex { } } +impl From for u16 { + fn from(value: QuarterIndex) -> Self { + value.0 + } +} + impl From for usize { fn from(value: QuarterIndex) -> Self { value.0 as usize @@ -51,6 +60,27 @@ impl Add for QuarterIndex { } } +impl Add for QuarterIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self::from(self.0 + rhs.0) + } +} + +impl AddAssign for QuarterIndex { + fn add_assign(&mut self, rhs: Self) { + *self = Self(self.0 + rhs.0) + } +} + +impl Div for QuarterIndex { + type Output = Self; + fn div(self, _: usize) -> Self::Output { + unreachable!() + } +} + impl From for QuarterIndex { fn from(value: MonthIndex) -> Self { Self((usize::from(value) / 3) as u16) diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs index 129be399b..81c00d7ea 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -36,9 +36,24 @@ pub struct Sats(u64); #[allow(clippy::inconsistent_digit_grouping)] impl Sats { pub const ZERO: Self = Self(0); - pub const MAX: Self = Self(u64::MAX); + pub const _1: Self = Self(1); + pub const _10: Self = Self(10); + pub const _100: Self = Self(100); + pub const _1K: Self = Self(1_000); + pub const _10K: Self = Self(10_000); + pub const _100K: Self = Self(100_000); + pub const _1M: Self = Self(1_000_000); + pub const _10M: Self = Self(10_000_000); + pub const _1BTC: Self = Self::ONE_BTC; + pub const _10BTC: Self = Self(10_00_000_000); + pub const _100BTC: Self = Self(100_00_000_000); + pub const _1K_BTC: Self = Self(1_000_00_000_000); + pub const _10K_BTC: Self = Self(10_000_00_000_000); + pub const _100K_BTC: Self = Self(100_000_00_000_000); pub const ONE_BTC: Self = Self(1_00_000_000); + pub const MAX: Self = Self(u64::MAX); pub const FIFTY_BTC: Self = Self(50_00_000_000); + pub const ONE_BTC_U128: u128 = 1_00_000_000; pub fn new(sats: u64) -> Self { Self(sats) @@ -81,6 +96,10 @@ impl CheckedSub for Sats { impl SubAssign for Sats { fn sub_assign(&mut self, rhs: Self) { *self = self.checked_sub(rhs).unwrap(); + // .unwrap_or_else(|| { + // dbg!((*self, rhs)); + // unreachable!(); + // }); } } diff --git a/crates/brk_core/src/structs/semesterindex.rs b/crates/brk_core/src/structs/semesterindex.rs new file mode 100644 index 000000000..4d3c0f649 --- /dev/null +++ b/crates/brk_core/src/structs/semesterindex.rs @@ -0,0 +1,104 @@ +use std::{ + fmt::Debug, + ops::{Add, AddAssign, Div}, +}; + +use serde::{Deserialize, Serialize}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::{CheckedSub, Printable}; + +use super::MonthIndex; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct SemesterIndex(u16); + +impl From for SemesterIndex { + fn from(value: u16) -> Self { + Self(value) + } +} + +impl From for SemesterIndex { + fn from(value: usize) -> Self { + Self(value as u16) + } +} + +impl From for u16 { + fn from(value: SemesterIndex) -> Self { + value.0 + } +} + +impl From for usize { + fn from(value: SemesterIndex) -> Self { + value.0 as usize + } +} + +impl Add for SemesterIndex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self::from(self.0 + rhs as u16) + } +} + +impl Add for SemesterIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self::from(self.0 + rhs.0) + } +} + +impl AddAssign for SemesterIndex { + fn add_assign(&mut self, rhs: Self) { + *self = Self(self.0 + rhs.0) + } +} + +impl Div for SemesterIndex { + type Output = Self; + fn div(self, _: usize) -> Self::Output { + unreachable!() + } +} + +impl From for SemesterIndex { + fn from(value: MonthIndex) -> Self { + Self((usize::from(value) / 6) as u16) + } +} + +impl CheckedSub for SemesterIndex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} + +impl Printable for SemesterIndex { + fn to_string() -> &'static str { + "semesterindex" + } + + fn to_possible_strings() -> &'static [&'static str] { + &["s", "semester", "semesterindex"] + } +} diff --git a/crates/brk_core/src/structs/stored_f32.rs b/crates/brk_core/src/structs/stored_f32.rs index 8da7e21ad..a50799a93 100644 --- a/crates/brk_core/src/structs/stored_f32.rs +++ b/crates/brk_core/src/structs/stored_f32.rs @@ -1,7 +1,7 @@ use core::panic; use std::{ cmp::Ordering, - ops::{Add, Div, Mul, Sub}, + ops::{Add, AddAssign, Div, Mul, Sub}, }; use derive_deref::Deref; @@ -70,6 +70,12 @@ impl Add for StoredF32 { } } +impl AddAssign for StoredF32 { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl From for f32 { fn from(value: StoredF32) -> Self { value.0 diff --git a/crates/brk_core/src/structs/stored_f64.rs b/crates/brk_core/src/structs/stored_f64.rs index 90c89bb9e..f22ff993a 100644 --- a/crates/brk_core/src/structs/stored_f64.rs +++ b/crates/brk_core/src/structs/stored_f64.rs @@ -1,7 +1,7 @@ use std::{ cmp::Ordering, f64, - ops::{Add, Div, Mul}, + ops::{Add, AddAssign, Div, Mul}, }; use derive_deref::Deref; @@ -65,6 +65,12 @@ impl Add for StoredF64 { } } +impl AddAssign for StoredF64 { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl From for f64 { fn from(value: StoredF64) -> Self { value.0 diff --git a/crates/brk_core/src/structs/stored_u32.rs b/crates/brk_core/src/structs/stored_u32.rs index de9569069..b1e4b5d96 100644 --- a/crates/brk_core/src/structs/stored_u32.rs +++ b/crates/brk_core/src/structs/stored_u32.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, AddAssign, Div}; use derive_deref::Deref; use serde::Serialize; @@ -73,6 +73,12 @@ impl Add for StoredU32 { } } +impl AddAssign for StoredU32 { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl From for StoredU32 { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { diff --git a/crates/brk_core/src/structs/stored_u64.rs b/crates/brk_core/src/structs/stored_u64.rs index 1edd718d8..82a8908dc 100644 --- a/crates/brk_core/src/structs/stored_u64.rs +++ b/crates/brk_core/src/structs/stored_u64.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, AddAssign, Div}; use derive_deref::Deref; use serde::Serialize; @@ -65,6 +65,12 @@ impl Add for StoredU64 { } } +impl AddAssign for StoredU64 { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl From for StoredU64 { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { diff --git a/crates/brk_core/src/structs/stored_u8.rs b/crates/brk_core/src/structs/stored_u8.rs index 8597589c3..0d36823aa 100644 --- a/crates/brk_core/src/structs/stored_u8.rs +++ b/crates/brk_core/src/structs/stored_u8.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, AddAssign, Div}; use derive_deref::Deref; use serde::Serialize; @@ -66,6 +66,12 @@ impl Add for StoredU8 { } } +impl AddAssign for StoredU8 { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl From for StoredU8 { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { diff --git a/crates/brk_core/src/structs/stored_usize.rs b/crates/brk_core/src/structs/stored_usize.rs index 547ed8ed7..18bd22749 100644 --- a/crates/brk_core/src/structs/stored_usize.rs +++ b/crates/brk_core/src/structs/stored_usize.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, AddAssign, Div}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; @@ -45,6 +45,11 @@ impl From for StoredUsize { Self(value) } } +impl From for usize { + fn from(value: StoredUsize) -> Self { + *value + } +} impl CheckedSub for StoredUsize { fn checked_sub(self, rhs: Self) -> Option { @@ -66,6 +71,12 @@ impl Add for StoredUsize { } } +impl AddAssign for StoredUsize { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl From for StoredUsize { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { diff --git a/crates/brk_core/src/structs/timestamp.rs b/crates/brk_core/src/structs/timestamp.rs index b5b733eea..67d74dbd6 100644 --- a/crates/brk_core/src/structs/timestamp.rs +++ b/crates/brk_core/src/structs/timestamp.rs @@ -1,7 +1,4 @@ -use std::{ - cmp::Ordering, - ops::{Add, Div}, -}; +use std::ops::{Add, AddAssign, Div}; use derive_deref::Deref; use jiff::{civil::date, tz::TimeZone}; @@ -29,7 +26,9 @@ use super::Date; )] pub struct Timestamp(u32); -const ONE_DAY_IN_SEC: i64 = 24 * 60 * 60; +const ONE_HOUR_IN_SEC: u32 = 60 * 60; +const ONE_DAY_IN_SEC: u32 = 24 * 60 * 60; +const ONE_DAY_IN_SEC_F64: f64 = ONE_DAY_IN_SEC as f64; impl Timestamp { pub const ZERO: Self = Self(0); @@ -50,30 +49,25 @@ impl Timestamp { Self::from(trunc_date_time.to_zoned(TimeZone::UTC).unwrap().timestamp()) } - pub fn difference_in_days_between(&self, other: Self) -> usize { - match self.cmp(&other) { - Ordering::Equal => 0, - Ordering::Greater => other.difference_in_days_between(*self), - Ordering::Less => { - (jiff::Timestamp::from(*self) - .duration_until(jiff::Timestamp::from(other)) - .as_secs() - / ONE_DAY_IN_SEC) as usize - } - } + #[inline] + pub fn difference_in_days_between(&self, older: Self) -> usize { + // if self.0 < older.0 { + // unreachable!() + // } + ((self.0 - older.0) / ONE_DAY_IN_SEC) as usize } - pub fn difference_in_days_between_float(&self, other: Self) -> f64 { - match self.cmp(&other) { - Ordering::Equal => 0.0, - Ordering::Greater => other.difference_in_days_between_float(*self), - Ordering::Less => { - jiff::Timestamp::from(*self) - .duration_until(jiff::Timestamp::from(other)) - .as_secs() as f64 - / ONE_DAY_IN_SEC as f64 - } - } + #[inline] + pub fn difference_in_days_between_float(&self, older: Self) -> f64 { + // if self.0 < older.0 { + // unreachable!() + // } + (self.0 - older.0) as f64 / ONE_DAY_IN_SEC_F64 + } + + #[inline] + pub fn is_more_than_hour(&self) -> bool { + self.0 >= ONE_HOUR_IN_SEC } } @@ -138,6 +132,12 @@ impl Add for Timestamp { } } +impl AddAssign for Timestamp { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl From for Timestamp { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { diff --git a/crates/brk_core/src/structs/typeindex.rs b/crates/brk_core/src/structs/typeindex.rs index 995e91d0a..8ea03066a 100644 --- a/crates/brk_core/src/structs/typeindex.rs +++ b/crates/brk_core/src/structs/typeindex.rs @@ -45,6 +45,11 @@ impl From for TypeIndex { Self(value) } } +impl From for u32 { + fn from(value: TypeIndex) -> Self { + value.0 + } +} impl From for TypeIndex { fn from(value: u64) -> Self { diff --git a/crates/brk_core/src/structs/typeindex_with_outputindex.rs b/crates/brk_core/src/structs/typeindex_with_outputindex.rs new file mode 100644 index 000000000..8a80a1dfd --- /dev/null +++ b/crates/brk_core/src/structs/typeindex_with_outputindex.rs @@ -0,0 +1,48 @@ +use byteview::ByteView; +use serde::Serialize; + +use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes}; + +use super::OutputIndex; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)] +pub struct TypeIndexWithOutputindex { + typeindex: TypeIndex, + outputindex: OutputIndex, +} + +impl From<(TypeIndex, OutputIndex)> for TypeIndexWithOutputindex { + fn from(value: (TypeIndex, OutputIndex)) -> Self { + Self { + typeindex: value.0, + outputindex: value.1, + } + } +} + +impl From for TypeIndexWithOutputindex { + fn from(value: ByteView) -> Self { + let typeindex = TypeIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap())); + let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap())); + Self { + typeindex, + outputindex, + } + } +} +impl From for ByteView { + fn from(value: TypeIndexWithOutputindex) -> Self { + ByteView::from(&value) + } +} +impl From<&TypeIndexWithOutputindex> for ByteView { + fn from(value: &TypeIndexWithOutputindex) -> Self { + ByteView::from( + [ + u32::from(value.typeindex).to_be_bytes().as_slice(), + u64::from(value.outputindex).to_be_bytes().as_slice(), + ] + .concat(), + ) + } +} diff --git a/crates/brk_core/src/structs/version.rs b/crates/brk_core/src/structs/version.rs index dc740367c..e05109b98 100644 --- a/crates/brk_core/src/structs/version.rs +++ b/crates/brk_core/src/structs/version.rs @@ -65,6 +65,12 @@ impl Version { } } +impl From for u64 { + fn from(value: Version) -> u64 { + value.0 + } +} + impl From for Version { fn from(value: u64) -> Self { Self(value) diff --git a/crates/brk_core/src/structs/weekindex.rs b/crates/brk_core/src/structs/weekindex.rs index 811ec4feb..d302cdfe7 100644 --- a/crates/brk_core/src/structs/weekindex.rs +++ b/crates/brk_core/src/structs/weekindex.rs @@ -1,4 +1,7 @@ -use std::{fmt::Debug, ops::Add}; +use std::{ + fmt::Debug, + ops::{Add, AddAssign, Div}, +}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -31,6 +34,12 @@ impl From for WeekIndex { } } +impl From for u16 { + fn from(value: WeekIndex) -> Self { + value.0 + } +} + impl From for WeekIndex { fn from(value: usize) -> Self { Self(value as u16) @@ -43,6 +52,27 @@ impl From for usize { } } +impl Add for WeekIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self::from(self.0 + rhs.0) + } +} + +impl AddAssign for WeekIndex { + fn add_assign(&mut self, rhs: Self) { + *self = Self(self.0 + rhs.0) + } +} + +impl Div for WeekIndex { + type Output = Self; + fn div(self, _: usize) -> Self::Output { + unreachable!() + } +} + impl Add for WeekIndex { type Output = Self; @@ -68,7 +98,6 @@ impl From for WeekIndex { let d = jiff::civil::Date::new(year, 6, 6).unwrap(); let i = d.iso_week_date(); let w = i.weeks_in_year(); - // dbg!(d, w); week += w as u16; year += 1; } diff --git a/crates/brk_core/src/structs/weight.rs b/crates/brk_core/src/structs/weight.rs index 5c56ceb6f..10b8ed8ad 100644 --- a/crates/brk_core/src/structs/weight.rs +++ b/crates/brk_core/src/structs/weight.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, AddAssign, Div}; use derive_deref::Deref; use serde::Serialize; @@ -58,6 +58,12 @@ impl Add for Weight { } } +impl AddAssign for Weight { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs + } +} + impl Div for Weight { type Output = Self; fn div(self, rhs: usize) -> Self::Output { diff --git a/crates/brk_core/src/structs/yearindex.rs b/crates/brk_core/src/structs/yearindex.rs index 78261f78f..8ba72883f 100644 --- a/crates/brk_core/src/structs/yearindex.rs +++ b/crates/brk_core/src/structs/yearindex.rs @@ -1,4 +1,7 @@ -use std::{fmt::Debug, ops::Add}; +use std::{ + fmt::Debug, + ops::{Add, AddAssign, Div}, +}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -51,6 +54,27 @@ impl Add for YearIndex { } } +impl Add for YearIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self::from(self.0 + rhs.0) + } +} + +impl AddAssign for YearIndex { + fn add_assign(&mut self, rhs: Self) { + *self = Self(self.0 + rhs.0) + } +} + +impl Div for YearIndex { + type Output = Self; + fn div(self, _: usize) -> Self::Output { + unreachable!() + } +} + impl From for YearIndex { fn from(value: DateIndex) -> Self { Self::from(Date::from(value)) diff --git a/crates/brk_core/src/traits/from_coarser.rs b/crates/brk_core/src/traits/from_coarser.rs new file mode 100644 index 000000000..93c1f697e --- /dev/null +++ b/crates/brk_core/src/traits/from_coarser.rs @@ -0,0 +1,18 @@ +use std::ops::RangeInclusive; + +pub trait FromCoarserIndex +where + T: Ord + From, +{ + fn min_from(coarser: T) -> usize; + fn max_from_(coarser: T) -> usize; + fn max_from(coarser: T, len: usize) -> usize { + Self::max_from_(coarser).min(len - 1) + } + fn inclusive_range_from(coarser: T, len: usize) -> RangeInclusive + where + T: Clone, + { + Self::min_from(coarser.clone())..=Self::max_from(coarser, len) + } +} diff --git a/crates/brk_core/src/traits/mod.rs b/crates/brk_core/src/traits/mod.rs index ca46c3328..8e724b8e7 100644 --- a/crates/brk_core/src/traits/mod.rs +++ b/crates/brk_core/src/traits/mod.rs @@ -1,3 +1,5 @@ +mod from_coarser; mod printable; +pub use from_coarser::*; pub use printable::*; diff --git a/crates/brk_core/src/utils/bytes.rs b/crates/brk_core/src/utils/bytes.rs index e1146adcc..52ea86924 100644 --- a/crates/brk_core/src/utils/bytes.rs +++ b/crates/brk_core/src/utils/bytes.rs @@ -1,5 +1,18 @@ use crate::{Error, Result}; +#[allow(clippy::result_unit_err)] +pub fn copy_first_4bytes(slice: &[u8]) -> Result<[u8; 4]> { + let mut buf: [u8; 4] = [0; 4]; + let buf_len = buf.len(); + if slice.len() < buf_len { + return Err(Error::String("Buffer is too small to convert to 8 bytes")); + } + slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { + buf[i] = *r; + }); + Ok(buf) +} + #[allow(clippy::result_unit_err)] pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> { let mut buf: [u8; 8] = [0; 8]; diff --git a/crates/brk_core/src/utils/rlimit.rs b/crates/brk_core/src/utils/rlimit.rs index 162a52574..f725c7b96 100644 --- a/crates/brk_core/src/utils/rlimit.rs +++ b/crates/brk_core/src/utils/rlimit.rs @@ -7,7 +7,7 @@ pub fn setrlimit() -> io::Result<()> { rlimit::setrlimit( Resource::NOFILE, - no_file_limit.0.max(210_000), + no_file_limit.0.max(10_000), no_file_limit.1, )?; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 876e445db..4d1e1ae0a 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -7,14 +7,15 @@ use std::{collections::BTreeMap, path::Path, str::FromStr, thread}; use brk_core::{ AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex, - OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, Version, Vin, Vout, - setrlimit, + OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, + Unit, Version, Vin, Vout, setrlimit, }; use bitcoin::{Transaction, TxIn, TxOut}; use brk_exit::Exit; use brk_parser::Parser; -use brk_vec::{AnyVec, VecIterator}; +use brk_store::AnyStore; +use brk_vec::{AnyVec, Mmap, VecIterator}; use color_eyre::eyre::{ContextCompat, eyre}; use log::{error, info}; use rayon::prelude::*; @@ -39,6 +40,7 @@ pub struct Indexer { impl Indexer { pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result { setrlimit()?; + Ok(Self { vecs: Vecs::forced_import(&outputs_dir.join("vecs/indexed"), VERSION + Version::ZERO)?, stores: Stores::forced_import(&outputs_dir.join("stores"), VERSION + Version::ZERO)?, @@ -55,6 +57,9 @@ impl Indexer { let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)) .unwrap_or_else(|_report| Indexes::default()); + // dbg!(starting_indexes); + // panic!(); + exit.block(); self.stores .rollback_if_needed(&mut self.vecs, &starting_indexes)?; @@ -84,9 +89,9 @@ impl Indexer { height: Height, rem: bool, exit: &Exit| - -> color_eyre::Result<()> { + -> color_eyre::Result { if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem || exit.triggered() { - return Ok(()); + return Ok(false); } info!("Exporting..."); @@ -94,15 +99,88 @@ impl Indexer { stores.commit(height)?; vecs.flush(height)?; exit.release(); - Ok(()) + Ok(true) }; + let mut txindex_to_first_outputindex_mmap_opt = None; + let mut p2pk65addressindex_to_p2pk65bytes_mmap_opt = None; + let mut p2pk33addressindex_to_p2pk33bytes_mmap_opt = None; + let mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt = None; + let mut p2shaddressindex_to_p2shbytes_mmap_opt = None; + let mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt = None; + let mut p2wshaddressindex_to_p2wshbytes_mmap_opt = None; + let mut p2traddressindex_to_p2trbytes_mmap_opt = None; + let mut p2aaddressindex_to_p2abytes_mmap_opt = None; + + let reset_mmaps_options = + |vecs: &mut Vecs, + txindex_to_first_outputindex_mmap_opt: &mut Option, + p2pk65addressindex_to_p2pk65bytes_mmap_opt: &mut Option, + p2pk33addressindex_to_p2pk33bytes_mmap_opt: &mut Option, + p2pkhaddressindex_to_p2pkhbytes_mmap_opt: &mut Option, + p2shaddressindex_to_p2shbytes_mmap_opt: &mut Option, + p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt: &mut Option, + p2wshaddressindex_to_p2wshbytes_mmap_opt: &mut Option, + p2traddressindex_to_p2trbytes_mmap_opt: &mut Option, + p2aaddressindex_to_p2abytes_mmap_opt: &mut Option| { + txindex_to_first_outputindex_mmap_opt + .replace(vecs.txindex_to_first_outputindex.create_mmap().unwrap()); + p2pk65addressindex_to_p2pk65bytes_mmap_opt.replace( + vecs.p2pk65addressindex_to_p2pk65bytes + .create_mmap() + .unwrap(), + ); + p2pk33addressindex_to_p2pk33bytes_mmap_opt.replace( + vecs.p2pk33addressindex_to_p2pk33bytes + .create_mmap() + .unwrap(), + ); + p2pkhaddressindex_to_p2pkhbytes_mmap_opt + .replace(vecs.p2pkhaddressindex_to_p2pkhbytes.create_mmap().unwrap()); + p2shaddressindex_to_p2shbytes_mmap_opt + .replace(vecs.p2shaddressindex_to_p2shbytes.create_mmap().unwrap()); + p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.replace( + vecs.p2wpkhaddressindex_to_p2wpkhbytes + .create_mmap() + .unwrap(), + ); + p2wshaddressindex_to_p2wshbytes_mmap_opt + .replace(vecs.p2wshaddressindex_to_p2wshbytes.create_mmap().unwrap()); + p2traddressindex_to_p2trbytes_mmap_opt + .replace(vecs.p2traddressindex_to_p2trbytes.create_mmap().unwrap()); + p2aaddressindex_to_p2abytes_mmap_opt + .replace(vecs.p2aaddressindex_to_p2abytes.create_mmap().unwrap()); + }; + + reset_mmaps_options( + vecs, + &mut txindex_to_first_outputindex_mmap_opt, + &mut p2pk65addressindex_to_p2pk65bytes_mmap_opt, + &mut p2pk33addressindex_to_p2pk33bytes_mmap_opt, + &mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt, + &mut p2shaddressindex_to_p2shbytes_mmap_opt, + &mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt, + &mut p2wshaddressindex_to_p2wshbytes_mmap_opt, + &mut p2traddressindex_to_p2trbytes_mmap_opt, + &mut p2aaddressindex_to_p2abytes_mmap_opt, + ); + parser.parse(start, end).iter().try_for_each( |(height, block, blockhash)| -> color_eyre::Result<()> { info!("Indexing block {height}..."); idxs.height = height; + let txindex_to_first_outputindex_mmap = txindex_to_first_outputindex_mmap_opt.as_ref().unwrap(); + let p2pk65addressindex_to_p2pk65bytes_mmap = p2pk65addressindex_to_p2pk65bytes_mmap_opt.as_ref().unwrap(); + let p2pk33addressindex_to_p2pk33bytes_mmap = p2pk33addressindex_to_p2pk33bytes_mmap_opt.as_ref().unwrap(); + let p2pkhaddressindex_to_p2pkhbytes_mmap = p2pkhaddressindex_to_p2pkhbytes_mmap_opt.as_ref().unwrap(); + let p2shaddressindex_to_p2shbytes_mmap = p2shaddressindex_to_p2shbytes_mmap_opt.as_ref().unwrap(); + let p2wpkhaddressindex_to_p2wpkhbytes_mmap = p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.as_ref().unwrap(); + let p2wshaddressindex_to_p2wshbytes_mmap = p2wshaddressindex_to_p2wshbytes_mmap_opt.as_ref().unwrap(); + let p2traddressindex_to_p2trbytes_mmap = p2traddressindex_to_p2trbytes_mmap_opt.as_ref().unwrap(); + let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_mmap_opt.as_ref().unwrap(); + // Used to check rapidhash collisions let check_collisions = check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO); @@ -132,34 +210,6 @@ impl Indexer { vecs.height_to_total_size.push_if_needed(height, block.total_size().into())?; vecs.height_to_weight.push_if_needed(height, block.weight().into())?; - let inputs = block - .txdata - .iter() - .enumerate() - .flat_map(|(index, tx)| { - tx.input - .iter() - .enumerate() - .map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx)) - }) - .collect::>(); - - let outputs = block - .txdata - .iter() - .enumerate() - .flat_map(|(index, tx)| { - tx.output - .iter() - .enumerate() - .map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx)) - }) - .collect::>(); - - let tx_len = block.txdata.len(); - let outputs_len = outputs.len(); - let inputs_len = inputs.len(); - let ( txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle, input_source_vec_handle, @@ -169,9 +219,9 @@ impl Indexer { scope.spawn(|| -> color_eyre::Result<_> { block .txdata - .par_iter() + .iter() .enumerate() - .map(|(index, tx)| -> color_eyre::Result<_> { + .map(|(index, tx)| { let txid = Txid::from(tx.compute_txid()); let txid_prefix = TxidPrefix::from(&txid); @@ -186,26 +236,21 @@ impl Indexer { Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt))) }) - .try_fold(BTreeMap::new, |mut map, tuple| { - let (key, value) = tuple?; - map.insert(key, value); - Ok(map) - }) - .try_reduce(BTreeMap::new, |mut map, mut map2| { - if map.len() > map2.len() { - map.append(&mut map2); - Ok(map) - } else { - map2.append(&mut map); - Ok(map2) - } - }) + .collect::>>() }); - let input_source_vec_handle = scope.spawn(|| { - let txindex_to_first_outputindex_mmap = vecs - .txindex_to_first_outputindex.mmap().load(); + let inputs = block + .txdata + .iter() + .enumerate() + .flat_map(|(index, tx)| { + tx.input + .iter() + .enumerate() + .map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx)) + }) + .collect::>(); inputs .into_par_iter() @@ -237,11 +282,11 @@ impl Indexer { let vout = Vout::from(outpoint.vout); - let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, &txindex_to_first_outputindex_mmap)? + let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_mmap)? .context("Expect outputindex to not be none") .inspect_err(|_| { dbg!(outpoint.txid, prev_txindex, vout); - })?.into_inner() + })?.into_owned() + vout; Ok((inputindex, InputSource::PreviousBlock(( @@ -266,157 +311,155 @@ impl Indexer { }) }); - let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| { - let p2pk65addressindex_to_p2pk65bytes_mmap = vecs - .p2pk65addressindex_to_p2pk65bytes.mmap().load(); - let p2pk33addressindex_to_p2pk33bytes_mmap = vecs.p2pk33addressindex_to_p2pk33bytes.mmap().load(); - let p2pkhaddressindex_to_p2pkhbytes_mmap = vecs.p2pkhaddressindex_to_p2pkhbytes.mmap().load(); - let p2shaddressindex_to_p2shbytes_mmap = vecs.p2shaddressindex_to_p2shbytes.mmap().load(); - let p2wpkhaddressindex_to_p2wpkhbytes_mmap = vecs.p2wpkhaddressindex_to_p2wpkhbytes.mmap().load(); - let p2wshaddressindex_to_p2wshbytes_mmap = vecs.p2wshaddressindex_to_p2wshbytes.mmap().load(); - let p2traddressindex_to_p2trbytes_mmap = vecs.p2traddressindex_to_p2trbytes.mmap().load(); - let p2aaddressindex_to_p2abytes_mmap = vecs.p2aaddressindex_to_p2abytes.mmap().load(); + let outputs = block + .txdata + .iter() + .enumerate() + .flat_map(|(index, tx)| { + tx.output + .iter() + .enumerate() + .map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx)) + }).collect::>(); - outputs - .into_par_iter() - .enumerate() - .map( - #[allow(clippy::type_complexity)] - |(block_outputindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<( - OutputIndex, - ( - &TxOut, - TxIndex, - Vout, - OutputType, - brk_core::Result, - Option, - &Transaction, - ), - )> { - let txindex = idxs.txindex + block_txindex; - let outputindex = idxs.outputindex + OutputIndex::from(block_outputindex); + let outputindex_to_txout_outputtype_addressbytes_res_addressindex = outputs.into_par_iter() + .enumerate() + .map( + #[allow(clippy::type_complexity)] + |(block_outputindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<( + OutputIndex, + ( + &TxOut, + TxIndex, + Vout, + OutputType, + brk_core::Result, + Option, + &Transaction, + ), + )> { + let txindex = idxs.txindex + block_txindex; + let outputindex = idxs.outputindex + OutputIndex::from(block_outputindex); - let script = &txout.script_pubkey; + let script = &txout.script_pubkey; - let outputtype = OutputType::from(script); + let outputtype = OutputType::from(script); - let address_bytes_res = - AddressBytes::try_from((script, outputtype)).inspect_err(|_| { - // dbg!(&txout, height, txi, &tx.compute_txid()); - }); - - let typeindex_opt = address_bytes_res.as_ref().ok().and_then(|addressbytes| { - stores - .addressbyteshash_to_typeindex - .get(&AddressBytesHash::from((addressbytes, outputtype))) - .unwrap() - .map(|v| *v) - // Checking if not in the future - .and_then(|typeindex_local| { - (typeindex_local < idxs.typeindex(outputtype)).then_some(typeindex_local) - }) + let address_bytes_res = + AddressBytes::try_from((script, outputtype)).inspect_err(|_| { + // dbg!(&txout, height, txi, &tx.compute_txid()); }); - if let Some(Some(typeindex)) = check_collisions.then_some(typeindex_opt) { - let addressbytes = address_bytes_res.as_ref().unwrap(); + let typeindex_opt = address_bytes_res.as_ref().ok().and_then(|addressbytes| { + stores + .addressbyteshash_to_typeindex + .get(&AddressBytesHash::from((addressbytes, outputtype))) + .unwrap() + .map(|v| *v) + // Checking if not in the future + .and_then(|typeindex_local| { + (typeindex_local < idxs.typeindex(outputtype)).then_some(typeindex_local) + }) + }); - let prev_addressbytes_opt = match outputtype { - OutputType::P2PK65 => vecs - .p2pk65addressindex_to_p2pk65bytes - .get_or_read(typeindex.into(), &p2pk65addressindex_to_p2pk65bytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2PK33 => vecs - .p2pk33addressindex_to_p2pk33bytes - .get_or_read(typeindex.into(), &p2pk33addressindex_to_p2pk33bytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2PKH => vecs - .p2pkhaddressindex_to_p2pkhbytes - .get_or_read(typeindex.into(), &p2pkhaddressindex_to_p2pkhbytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2SH => vecs - .p2shaddressindex_to_p2shbytes - .get_or_read(typeindex.into(), &p2shaddressindex_to_p2shbytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2WPKH => vecs - .p2wpkhaddressindex_to_p2wpkhbytes - .get_or_read(typeindex.into(), &p2wpkhaddressindex_to_p2wpkhbytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2WSH => vecs - .p2wshaddressindex_to_p2wshbytes - .get_or_read(typeindex.into(), &p2wshaddressindex_to_p2wshbytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2TR => vecs - .p2traddressindex_to_p2trbytes - .get_or_read(typeindex.into(), &p2traddressindex_to_p2trbytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2A => vecs - .p2aaddressindex_to_p2abytes - .get_or_read(typeindex.into(), &p2aaddressindex_to_p2abytes_mmap)? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => { - unreachable!() - } - }; - let prev_addressbytes = - prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?; + if let Some(Some(typeindex)) = check_collisions.then_some(typeindex_opt) { + let addressbytes = address_bytes_res.as_ref().unwrap(); - if stores.addressbyteshash_to_typeindex.needs(height) - && prev_addressbytes != addressbytes - { - let txid = tx.compute_txid(); - dbg!( - height, - txid, - vout, - block_txindex, - outputtype, - prev_addressbytes, - addressbytes, - &idxs, - typeindex, - typeindex, - txout, - AddressBytesHash::from((addressbytes, outputtype)), - ); - panic!() + let prev_addressbytes_opt = match outputtype { + OutputType::P2PK65 => vecs + .p2pk65addressindex_to_p2pk65bytes + .get_or_read(typeindex.into(), p2pk65addressindex_to_p2pk65bytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::P2PK33 => vecs + .p2pk33addressindex_to_p2pk33bytes + .get_or_read(typeindex.into(), p2pk33addressindex_to_p2pk33bytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::P2PKH => vecs + .p2pkhaddressindex_to_p2pkhbytes + .get_or_read(typeindex.into(), p2pkhaddressindex_to_p2pkhbytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::P2SH => vecs + .p2shaddressindex_to_p2shbytes + .get_or_read(typeindex.into(), p2shaddressindex_to_p2shbytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::P2WPKH => vecs + .p2wpkhaddressindex_to_p2wpkhbytes + .get_or_read(typeindex.into(), p2wpkhaddressindex_to_p2wpkhbytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::P2WSH => vecs + .p2wshaddressindex_to_p2wshbytes + .get_or_read(typeindex.into(), p2wshaddressindex_to_p2wshbytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::P2TR => vecs + .p2traddressindex_to_p2trbytes + .get_or_read(typeindex.into(), p2traddressindex_to_p2trbytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::P2A => vecs + .p2aaddressindex_to_p2abytes + .get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_mmap)? + .map(|v| AddressBytes::from(v.into_owned())), + OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => { + unreachable!() } - } + }; + let prev_addressbytes = + prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?; - Ok(( - outputindex, - ( - txout, - txindex, + if stores.addressbyteshash_to_typeindex.needs(height) + && prev_addressbytes != addressbytes + { + let txid = tx.compute_txid(); + dbg!( + height, + txid, vout, + block_txindex, outputtype, - address_bytes_res, - typeindex_opt, - tx, - ), - )) - }, - ) - .try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> { - let (key, value) = tuple?; - map.insert(key, value); - Ok(map) - }) - .try_reduce(BTreeMap::new, |mut map, mut map2| { - if map.len() > map2.len() { - map.append(&mut map2); - Ok(map) - } else { - map2.append(&mut map); - Ok(map2) + prev_addressbytes, + addressbytes, + &idxs, + typeindex, + typeindex, + txout, + AddressBytesHash::from((addressbytes, outputtype)), + ); + panic!() + } } - }) - }); + + Ok(( + outputindex, + ( + txout, + txindex, + vout, + outputtype, + address_bytes_res, + typeindex_opt, + tx, + ), + )) + }, + ) + .try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> { + let (key, value) = tuple?; + map.insert(key, value); + Ok(map) + }) + .try_reduce(BTreeMap::new, |mut map, mut map2| { + if map.len() > map2.len() { + map.append(&mut map2); + Ok(map) + } else { + map2.append(&mut map); + Ok(map2) + } + }); ( txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(), input_source_vec_handle.join(), - outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle.join(), + outputindex_to_txout_outputtype_addressbytes_res_addressindex, ) }); @@ -436,7 +479,11 @@ impl Indexer { .ok() .context( "Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join", - )??; + )?; + + let outputs_len = outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt.len(); + let inputs_len = input_source_vec.len(); + let tx_len = block.txdata.len(); let mut new_txindexvout_to_outputindex: BTreeMap< (TxIndex, Vout), @@ -546,6 +593,10 @@ impl Indexer { new_txindexvout_to_outputindex .insert((txindex, vout), outputindex); + if outputtype.is_address() { + stores.addresstype_to_typeindex_with_outputindex.get_mut(outputtype).unwrap().insert_if_needed(TypeIndexWithOutputindex::from((typeindex, outputindex)), Unit, height); + } + Ok(()) }, )?; @@ -687,7 +738,22 @@ impl Indexer { idxs.inputindex += InputIndex::from(inputs_len); idxs.outputindex += OutputIndex::from(outputs_len); - export_if_needed(stores, vecs, height, false, exit)?; + let exported = export_if_needed(stores, vecs, height, false, exit)?; + + if exported { + reset_mmaps_options( + vecs, + &mut txindex_to_first_outputindex_mmap_opt, + &mut p2pk65addressindex_to_p2pk65bytes_mmap_opt, + &mut p2pk33addressindex_to_p2pk33bytes_mmap_opt, + &mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt, + &mut p2shaddressindex_to_p2shbytes_mmap_opt, + &mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt, + &mut p2wshaddressindex_to_p2wshbytes_mmap_opt, + &mut p2traddressindex_to_p2trbytes_mmap_opt, + &mut p2aaddressindex_to_p2abytes_mmap_opt, + ); + } Ok(()) }, @@ -695,8 +761,6 @@ impl Indexer { export_if_needed(stores, vecs, idxs.height, true, exit)?; - stores.rotate_memtables(); - Ok(starting_indexes) } } diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index 9ea1ef32c..ee7e69f48 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -1,12 +1,13 @@ -use std::{fs, path::Path, thread}; +use std::{borrow::Cow, fs, path::Path, thread}; use brk_core::{ - AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, Result, TxIndex, - TxidPrefix, TypeIndex, Value, Version, + AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex, + OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, }; -use brk_store::Store; -use brk_vec::AnyIterableVec; +use brk_store::{AnyStore, Store}; +use brk_vec::{AnyIterableVec, VecIterator}; use fjall::{PersistMode, TransactionalKeyspace}; +use rayon::prelude::*; use crate::Indexes; @@ -15,9 +16,12 @@ use super::Vecs; #[derive(Clone)] pub struct Stores { pub keyspace: TransactionalKeyspace, + pub addressbyteshash_to_typeindex: Store, pub blockhashprefix_to_height: Store, pub txidprefix_to_txindex: Store, + pub addresstype_to_typeindex_with_outputindex: + ByAddressType>, } const VERSION: Version = Version::ZERO; @@ -62,24 +66,193 @@ impl Stores { None, ) }); + let p2aaddressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2aaddressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); + let p2pk33addressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2pk33addressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); + let p2pk65addressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2pk65addressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); + let p2pkhaddressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2pkhaddressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); + let p2shaddressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2shaddressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); + let p2traddressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2traddressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); + let p2wpkhaddressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2wpkhaddressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); + let p2wshaddressindex_with_outputindex = scope.spawn(|| { + Store::import( + &keyspace, + path, + "p2wshaddressindex_with_outputindex", + version + VERSION + Version::ZERO, + Some(false), + ) + }); Ok(Self { keyspace: keyspace.clone(), + addressbyteshash_to_typeindex: addressbyteshash_to_typeindex.join().unwrap()?, blockhashprefix_to_height: blockhashprefix_to_height.join().unwrap()?, txidprefix_to_txindex: txidprefix_to_txindex.join().unwrap()?, + addresstype_to_typeindex_with_outputindex: ByAddressType { + p2pk65: p2pk65addressindex_with_outputindex.join().unwrap()?, + p2pk33: p2pk33addressindex_with_outputindex.join().unwrap()?, + p2pkh: p2pkhaddressindex_with_outputindex.join().unwrap()?, + p2sh: p2shaddressindex_with_outputindex.join().unwrap()?, + p2wpkh: p2wpkhaddressindex_with_outputindex.join().unwrap()?, + p2wsh: p2wshaddressindex_with_outputindex.join().unwrap()?, + p2tr: p2traddressindex_with_outputindex.join().unwrap()?, + p2a: p2aaddressindex_with_outputindex.join().unwrap()?, + }, }) }) } + pub fn starting_height(&self) -> Height { + self.as_slice() + .into_iter() + .map(|store| { + // let height = + store.height().map(Height::incremented).unwrap_or_default() + // dbg!((height, store.name())); + }) + .min() + .unwrap() + } + + pub fn commit(&mut self, height: Height) -> Result<()> { + self.as_mut_slice() + .into_par_iter() + .try_for_each(|store| store.commit(height))?; + + self.keyspace + .persist(PersistMode::SyncAll) + .map_err(|e| e.into()) + } + + fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 11] { + [ + &self.addressbyteshash_to_typeindex, + &self.blockhashprefix_to_height, + &self.txidprefix_to_txindex, + &self.addresstype_to_typeindex_with_outputindex.p2a, + &self.addresstype_to_typeindex_with_outputindex.p2pk33, + &self.addresstype_to_typeindex_with_outputindex.p2pk65, + &self.addresstype_to_typeindex_with_outputindex.p2pkh, + &self.addresstype_to_typeindex_with_outputindex.p2sh, + &self.addresstype_to_typeindex_with_outputindex.p2tr, + &self.addresstype_to_typeindex_with_outputindex.p2wpkh, + &self.addresstype_to_typeindex_with_outputindex.p2wsh, + ] + } + + fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 11] { + [ + &mut self.addressbyteshash_to_typeindex, + &mut self.blockhashprefix_to_height, + &mut self.txidprefix_to_txindex, + &mut self.addresstype_to_typeindex_with_outputindex.p2a, + &mut self.addresstype_to_typeindex_with_outputindex.p2pk33, + &mut self.addresstype_to_typeindex_with_outputindex.p2pk65, + &mut self.addresstype_to_typeindex_with_outputindex.p2pkh, + &mut self.addresstype_to_typeindex_with_outputindex.p2sh, + &mut self.addresstype_to_typeindex_with_outputindex.p2tr, + &mut self.addresstype_to_typeindex_with_outputindex.p2wpkh, + &mut self.addresstype_to_typeindex_with_outputindex.p2wsh, + ] + } + pub fn rollback_if_needed( &mut self, vecs: &mut Vecs, starting_indexes: &Indexes, ) -> color_eyre::Result<()> { - if self.addressbyteshash_to_typeindex.is_empty() - && self.blockhashprefix_to_height.is_empty() - && self.txidprefix_to_txindex.is_empty() + if self.addressbyteshash_to_typeindex.is_empty()? + && self.blockhashprefix_to_height.is_empty()? + && self.txidprefix_to_txindex.is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2a + .is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2pk33 + .is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2pk65 + .is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2pkh + .is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2sh + .is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2tr + .is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2wpkh + .is_empty()? + && self + .addresstype_to_typeindex_with_outputindex + .p2wsh + .is_empty()? { return Ok(()); } @@ -88,7 +261,7 @@ impl Stores { vecs.height_to_blockhash .iter_at(starting_indexes.height) .for_each(|(_, v)| { - let blockhashprefix = BlockHashPrefix::from(Value::into_inner(v)); + let blockhashprefix = BlockHashPrefix::from(v.into_owned()); self.blockhashprefix_to_height.remove(blockhashprefix); }); @@ -96,14 +269,14 @@ impl Stores { .height_to_first_p2pk65addressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2pk65addressindex_to_p2pk65bytes_iter = vecs.p2pk65addressindex_to_p2pk65bytes.iter(); while let Some(typedbytes) = p2pk65addressindex_to_p2pk65bytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2PK65)); @@ -116,14 +289,14 @@ impl Stores { .height_to_first_p2pk33addressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2pk33addressindex_to_p2pk33bytes_iter = vecs.p2pk33addressindex_to_p2pk33bytes.iter(); while let Some(typedbytes) = p2pk33addressindex_to_p2pk33bytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2PK33)); @@ -136,14 +309,14 @@ impl Stores { .height_to_first_p2pkhaddressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2pkhaddressindex_to_p2pkhbytes_iter = vecs.p2pkhaddressindex_to_p2pkhbytes.iter(); while let Some(typedbytes) = p2pkhaddressindex_to_p2pkhbytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2PKH)); @@ -156,14 +329,14 @@ impl Stores { .height_to_first_p2shaddressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2shaddressindex_to_p2shbytes_iter = vecs.p2shaddressindex_to_p2shbytes.iter(); while let Some(typedbytes) = p2shaddressindex_to_p2shbytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2SH)); @@ -176,14 +349,14 @@ impl Stores { .height_to_first_p2traddressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2traddressindex_to_p2trbytes_iter = vecs.p2traddressindex_to_p2trbytes.iter(); while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2TR)); @@ -196,14 +369,14 @@ impl Stores { .height_to_first_p2wpkhaddressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2wpkhaddressindex_to_p2wpkhbytes_iter = vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter(); while let Some(typedbytes) = p2wpkhaddressindex_to_p2wpkhbytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2WPKH)); @@ -216,14 +389,14 @@ impl Stores { .height_to_first_p2wshaddressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2wshaddressindex_to_p2wshbytes_iter = vecs.p2wshaddressindex_to_p2wshbytes.iter(); while let Some(typedbytes) = p2wshaddressindex_to_p2wshbytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2WSH)); @@ -236,13 +409,13 @@ impl Stores { .height_to_first_p2aaddressindex .iter() .get(starting_indexes.height) - .map(Value::into_inner) + .map(Cow::into_owned) { let mut p2aaddressindex_to_p2abytes_iter = vecs.p2aaddressindex_to_p2abytes.iter(); while let Some(typedbytes) = p2aaddressindex_to_p2abytes_iter .get(index) - .map(Value::into_inner) + .map(Cow::into_owned) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2A)); @@ -251,15 +424,15 @@ impl Stores { } } } else { - self.blockhashprefix_to_height.reset_partition()?; - self.addressbyteshash_to_typeindex.reset_partition()?; + self.blockhashprefix_to_height.reset()?; + self.addressbyteshash_to_typeindex.reset()?; } if starting_indexes.txindex != TxIndex::ZERO { vecs.txindex_to_txid .iter_at(starting_indexes.txindex) .for_each(|(txindex, txid)| { - let txidprefix = TxidPrefix::from(&txid.into_inner()); + let txidprefix = TxidPrefix::from(&txid.into_owned()); // "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599" let is_not_first_dup = txindex != TxIndex::new(142783) @@ -274,52 +447,51 @@ impl Stores { } }); } else { - self.txidprefix_to_txindex.reset_partition()?; + self.txidprefix_to_txindex.reset()?; + } + + if starting_indexes.outputindex != OutputIndex::ZERO { + let mut outputindex_to_typeindex_iter = vecs.outputindex_to_typeindex.into_iter(); + vecs.outputindex_to_outputtype + .iter_at(starting_indexes.outputindex) + .filter(|(_, outputtype)| outputtype.is_address()) + .for_each(|(outputindex, outputtype)| { + let outputtype = outputtype.into_owned(); + + let typeindex = outputindex_to_typeindex_iter.unwrap_get_inner(outputindex); + + self.addresstype_to_typeindex_with_outputindex + .get_mut(outputtype) + .unwrap() + .remove(TypeIndexWithOutputindex::from((typeindex, outputindex))); + }); + } else { + self.addresstype_to_typeindex_with_outputindex.p2a.reset()?; + self.addresstype_to_typeindex_with_outputindex + .p2pk33 + .reset()?; + self.addresstype_to_typeindex_with_outputindex + .p2pk65 + .reset()?; + self.addresstype_to_typeindex_with_outputindex + .p2pkh + .reset()?; + self.addresstype_to_typeindex_with_outputindex + .p2sh + .reset()?; + self.addresstype_to_typeindex_with_outputindex + .p2tr + .reset()?; + self.addresstype_to_typeindex_with_outputindex + .p2wpkh + .reset()?; + self.addresstype_to_typeindex_with_outputindex + .p2wsh + .reset()?; } self.commit(starting_indexes.height.decremented().unwrap_or_default())?; Ok(()) } - - pub fn starting_height(&self) -> Height { - [ - self.addressbyteshash_to_typeindex.height(), - self.blockhashprefix_to_height.height(), - self.txidprefix_to_txindex.height(), - ] - .into_iter() - .map(|height| height.map(Height::incremented).unwrap_or_default()) - .min() - .unwrap() - } - - pub fn commit(&mut self, height: Height) -> Result<()> { - thread::scope(|scope| -> Result<()> { - let addressbyteshash_to_typeindex_commit_handle = - scope.spawn(|| self.addressbyteshash_to_typeindex.commit(height)); - let blockhashprefix_to_height_commit_handle = - scope.spawn(|| self.blockhashprefix_to_height.commit(height)); - let txidprefix_to_txindex_commit_handle = - scope.spawn(|| self.txidprefix_to_txindex.commit(height)); - - addressbyteshash_to_typeindex_commit_handle - .join() - .unwrap()?; - blockhashprefix_to_height_commit_handle.join().unwrap()?; - txidprefix_to_txindex_commit_handle.join().unwrap()?; - - Ok(()) - })?; - - self.keyspace - .persist(PersistMode::SyncAll) - .map_err(|e| e.into()) - } - - pub fn rotate_memtables(&self) { - self.addressbyteshash_to_typeindex.rotate_memtable(); - self.blockhashprefix_to_height.rotate_memtable(); - self.txidprefix_to_txindex.rotate_memtable(); - } } diff --git a/crates/brk_interface/Cargo.toml b/crates/brk_interface/Cargo.toml index 8d50deb21..b3dc440c9 100644 --- a/crates/brk_interface/Cargo.toml +++ b/crates/brk_interface/Cargo.toml @@ -18,5 +18,5 @@ brk_rmcp = { workspace = true } schemars = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -serde_with = "3.13.0" +serde_with = "3.14.0" tabled = { workspace = true } diff --git a/crates/brk_interface/src/index.rs b/crates/brk_interface/src/index.rs index a92b8fd90..4eaf5bb7e 100644 --- a/crates/brk_interface/src/index.rs +++ b/crates/brk_interface/src/index.rs @@ -4,7 +4,8 @@ use brk_core::{ DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height, InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, - P2WSHAddressIndex, Printable, QuarterIndex, TxIndex, UnknownOutputIndex, WeekIndex, YearIndex, + P2WSHAddressIndex, Printable, QuarterIndex, SemesterIndex, TxIndex, UnknownOutputIndex, + WeekIndex, YearIndex, }; use color_eyre::eyre::eyre; use schemars::JsonSchema; @@ -52,6 +53,8 @@ pub enum Index { P2WSHAddressIndex, #[schemars(description = "Quarter index")] QuarterIndex, + #[schemars(description = "Semester index")] + SemesterIndex, #[schemars(description = "Transaction index")] TxIndex, #[schemars(description = "Unknown output index")] @@ -63,7 +66,7 @@ pub enum Index { } impl Index { - pub fn all() -> [Self; 24] { + pub fn all() -> [Self; 25] { [ Self::DateIndex, Self::DecadeIndex, @@ -85,6 +88,7 @@ impl Index { Self::P2WPKHAddressIndex, Self::P2WSHAddressIndex, Self::QuarterIndex, + Self::SemesterIndex, Self::TxIndex, Self::UnknownOutputIndex, Self::WeekIndex, @@ -114,6 +118,7 @@ impl Index { Self::P2WPKHAddressIndex => P2WPKHAddressIndex::to_possible_strings(), Self::P2WSHAddressIndex => P2WSHAddressIndex::to_possible_strings(), Self::QuarterIndex => QuarterIndex::to_possible_strings(), + Self::SemesterIndex => SemesterIndex::to_possible_strings(), Self::TxIndex => TxIndex::to_possible_strings(), Self::UnknownOutputIndex => UnknownOutputIndex::to_possible_strings(), Self::WeekIndex => WeekIndex::to_possible_strings(), @@ -174,7 +179,7 @@ impl TryFrom<&str> for Index { Self::P2WSHAddressIndex } v if (Self::QuarterIndex).possible_values().contains(&v) => Self::QuarterIndex, - v if (Self::QuarterIndex).possible_values().contains(&v) => Self::QuarterIndex, + v if (Self::SemesterIndex).possible_values().contains(&v) => Self::SemesterIndex, v if (Self::TxIndex).possible_values().contains(&v) => Self::TxIndex, v if (Self::WeekIndex).possible_values().contains(&v) => Self::WeekIndex, v if (Self::YearIndex).possible_values().contains(&v) => Self::YearIndex, @@ -188,7 +193,7 @@ impl TryFrom<&str> for Index { impl fmt::Display for Index { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index 59de7ea9d..e9bf3df43 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -6,9 +6,9 @@ use std::collections::BTreeMap; use brk_computer::Computer; -use brk_core::Result; +use brk_core::{Height, Result}; use brk_indexer::Indexer; -use brk_vec::AnyCollectableVec; +use brk_vec::{AnyCollectableVec, AnyIndexedVec}; use tabled::settings::Style; mod deser; @@ -46,6 +46,10 @@ impl<'a> Interface<'a> { } } + pub fn get_height(&self) -> Height { + self._indexer.vecs.height_to_blockhash.height() + } + pub fn search(&self, params: &Params) -> Vec<(String, &&dyn AnyCollectableVec)> { let tuples = params .ids @@ -85,9 +89,19 @@ impl<'a> Interface<'a> { vecs: Vec<(String, &&dyn AnyCollectableVec)>, params: &ParamsOpt, ) -> color_eyre::Result { - let from = params.from(); - let to = params.to(); - let format = params.format(); + let from = params.from().map(|from| { + vecs.iter() + .map(|(_, v)| v.i64_to_usize(from)) + .min() + .unwrap_or_default() + }); + + let to = params.to().map(|to| { + vecs.iter() + .map(|(_, v)| v.i64_to_usize(to)) + .min() + .unwrap_or_default() + }); let mut values = vecs .iter() @@ -96,6 +110,8 @@ impl<'a> Interface<'a> { }) .collect::>>()?; + let format = params.format(); + if values.is_empty() { return Ok(Output::default(format)); } diff --git a/crates/brk_interface/src/output.rs b/crates/brk_interface/src/output.rs index 4562ed66d..287e6bd23 100644 --- a/crates/brk_interface/src/output.rs +++ b/crates/brk_interface/src/output.rs @@ -36,9 +36,9 @@ impl fmt::Display for Output { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Json(value) => write!(f, "{}", serde_json::to_string_pretty(value).unwrap()), - Self::CSV(string) => write!(f, "{}", string), - Self::TSV(string) => write!(f, "{}", string), - Self::MD(string) => write!(f, "{}", string), + Self::CSV(string) => write!(f, "{string}"), + Self::TSV(string) => write!(f, "{string}"), + Self::MD(string) => write!(f, "{string}"), } } } diff --git a/crates/brk_interface/src/vecs.rs b/crates/brk_interface/src/vecs.rs index 1f4b65a68..93ea6c2c0 100644 --- a/crates/brk_interface/src/vecs.rs +++ b/crates/brk_interface/src/vecs.rs @@ -101,33 +101,33 @@ impl<'a> Vecs<'a> { fn insert(&mut self, vec: &'a dyn AnyCollectableVec) { let name = vec.name(); let serialized_index = vec.index_type_to_string(); - let split = name.split("_to_").collect::>(); - if split.len() != 1 - && !(split.len() == 2 - && split.first().is_some_and(|s| { - s == &"up" - || s == &"start" - || s.ends_with("relative") - || s.starts_with("from") - || s == &"cumulative_up" - || s.starts_with("cumulative_start") - || s.starts_with("cumulative_from") - || s == &"activity" - })) - && !(split.len() == 3 - && split.first().is_some_and(|s| { - s == &"up" - || s == &"start" - || s.starts_with("from") - || s == &"cumulative_up" - || s == &"cumulative_start" - || s.starts_with("cumulative_from") - }) - && split.get(1).is_some_and(|s| s.ends_with("relative"))) - { - dbg!((&serialized_index, &name, &split)); - unreachable!(); - } + // let split = name.split("_to_").collect::>(); + // if split.len() != 1 + // && !(split.len() == 2 + // && split.first().is_some_and(|s| { + // s == &"up" + // || s == &"start" + // || s.ends_with("relative") + // || s.starts_with("from") + // || s == &"cumulative_up" + // || s.starts_with("cumulative_start") + // || s.starts_with("cumulative_from") + // || s == &"activity" + // })) + // && !(split.len() == 3 + // && split.first().is_some_and(|s| { + // s == &"up" + // || s == &"start" + // || s.starts_with("from") + // || s == &"cumulative_up" + // || s == &"cumulative_start" + // || s.starts_with("cumulative_from") + // }) + // && split.get(1).is_some_and(|s| s.ends_with("relative"))) + // { + // dbg!((&serialized_index, &name, &split)); + // unreachable!(); + // } let index = Index::try_from(serialized_index) .inspect_err(|_| { dbg!(&serialized_index); diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index e95f7b465..c11b29ad8 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -14,7 +14,7 @@ use color_eyre::owo_colors::OwoColorize; use env_logger::{Builder, Env}; use jiff::{Timestamp, tz}; -#[inline(always)] +#[inline] pub fn init(path: Option<&Path>) { let file = path.map(|path| { let _ = fs::remove_file(path); @@ -34,7 +34,7 @@ pub fn init(path: Option<&Path>) { .strftime("%Y-%m-%d %H:%M:%S") .to_string(); let level = record.level().as_str().to_lowercase(); - let level = format!("{:5}", level); + let level = format!("{level:5}"); let target = record.target(); let dash = "-"; let args = record.args(); @@ -81,7 +81,7 @@ fn write( dash: impl Display, args: impl Display, ) -> Result<(), std::io::Error> { - writeln!(buf, "{} {} {} {}", date_time, dash, level, args) + writeln!(buf, "{date_time} {dash} {level} {args}") // Don't remove, used to know the target of unwanted logs // writeln!( // buf, diff --git a/crates/brk_mcp/Cargo.toml b/crates/brk_mcp/Cargo.toml index 6e63c2124..ec71f78ff 100644 --- a/crates/brk_mcp/Cargo.toml +++ b/crates/brk_mcp/Cargo.toml @@ -12,4 +12,3 @@ axum = { workspace = true } brk_interface = { workspace = true } log = { workspace = true } brk_rmcp = { workspace = true } -tracing = "0.1.41" diff --git a/crates/brk_mcp/README.md b/crates/brk_mcp/README.md index a4bef4581..7877a138f 100644 --- a/crates/brk_mcp/README.md +++ b/crates/brk_mcp/README.md @@ -2,33 +2,43 @@ A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK +## URLs + +- https://eu1.bitcoinresearchkit.org/mcp +- https://eu2.bitcoinresearchkit.org/mcp + ## Usage -To connect to the MCP you only need the correct URL ([https://bitcoinresearchkit.org/mcp](https://bitcoinresearchkit.org/mcp)) no token or auth is needed. +To connect to the MCP use any of the previous URL, no token or auth is needed. This implementation has only been tested with Claude and the [MCP inspector](https://modelcontextprotocol.io/docs/tools/inspector). -Please be aware thtat the technology is very rapidly evolving, thus having issues if probably expected. +Please be aware that the technology is evolving very rapidly, thus having issues is probably expected. If you, you can join the discord see if there is a solution. ### Claude #### Step 1 -First we need to connect BRK to Claude. +First we need to connect BRK to Claude. To do that we need to go to the "Connect apps" menu from the home screen of Claude desktop. -![Caude MCP setup step 1](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step1.png) +![Image of Claude Desktop home screen](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step1.png) #### Step 2 -Which is done by adding an integration. +Then simply go to "Add integration". -![Caude MCP setup step 2](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step2.png) +![Image of the Connect app" menu of Claude Desktop](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step2.png) #### Step 3 -Since BRK's open and free, only a URL is needed. +Claude's MCP client is (for now?) session based thus using a URL pointing to a load balancer will not work. -![Caude MCP setup step 3](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step3.png) +Use one of the following URL instead: + +- https://eu1.bitcoinresearchkit.org/mcp +- https://eu2.bitcoinresearchkit.org/mcp + +![Image of Add Integration menu of Claude Desktop](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step3.png) #### Step 4 @@ -36,12 +46,4 @@ Verify that it has access to BRK's tools. Optionally and highly recommended, giving it unsupervised access gives a more fluid experience and prevents possible issues and errors. -![Caude MCP setup step 4](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step4.png) - -#### Results - -Some examples of dashboard and documents generated by Claude using BRK's tools: - -- [Document](https://claude.ai/public/artifacts/71194d29-f965-417c-ba09-fdf0e4ecb1d5) -- [Dashboard](https://claude.ai/public/artifacts/beef143f-399a-4ed4-b8bf-c986b776de42) -- [Dashboard2](https://claude.ai/public/artifacts/5430ae49-bb3d-4fc1-ab24-f1e33deb40dc) +![Image of edit integration meny on Claude Desktop](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step4.png) diff --git a/crates/brk_parser/src/xor_index.rs b/crates/brk_parser/src/xor_index.rs index 81b79498d..0ce4cf682 100644 --- a/crates/brk_parser/src/xor_index.rs +++ b/crates/brk_parser/src/xor_index.rs @@ -17,14 +17,14 @@ impl XORIndex { bytes } - #[inline(always)] + #[inline] pub fn byte(&mut self, mut byte: u8, xor_bytes: &XORBytes) -> u8 { byte ^= xor_bytes[self.0]; self.increment(); byte } - #[inline(always)] + #[inline] pub fn increment(&mut self) { self.0 += 1; if self.0 == XOR_LEN { @@ -32,7 +32,7 @@ impl XORIndex { } } - #[inline(always)] + #[inline] pub fn add_assign(&mut self, i: usize) { self.0 = (self.0 + i) % XOR_LEN; } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 099f50d3a..50907bd44 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -32,7 +32,7 @@ serde_json = { workspace = true } tokio = { workspace = true } tower-http = { version = "0.6.6", features = ["compression-full", "trace"] } tracing = "0.1.41" -zip = "4.2.0" +zip = "4.3.0" [package.metadata.cargo-machete] ignored = ["clap"] diff --git a/crates/brk_server/README.md b/crates/brk_server/README.md index 2a0d632d0..9f92959e7 100644 --- a/crates/brk_server/README.md +++ b/crates/brk_server/README.md @@ -49,7 +49,7 @@ Get the count of all existing indexes. Get the count of all existing vec ids. -#### [`GET /api/vecs/variant-count`](https://bitcoinresearchkit.org/api/vecs/variant-count) +#### [`GET /api/vecs/vec-count`](https://bitcoinresearchkit.org/api/vecs/vec-count) Get the count of all existing vecs. \ Equals to the sum of supported Indexes of each vec id. diff --git a/crates/brk_server/src/api/interface/bridge.rs b/crates/brk_server/src/api/interface/bridge.rs index eac299108..5f0cd2bce 100644 --- a/crates/brk_server/src/api/interface/bridge.rs +++ b/crates/brk_server/src/api/interface/bridge.rs @@ -36,10 +36,9 @@ impl Bridge for Interface<'static> { // File auto-generated, any modifications will be overwritten // -export const VERSION = \"v{}\"; +export const VERSION = \"v{VERSION}\"; -", - VERSION +" ); contents += &indexes @@ -53,7 +52,7 @@ export const VERSION = \"v{}\"; .join("\n"); contents += &format!( - "\n\n/** @typedef {{{}}} Index */", + "\n\n/** @typedef {{{}}} Index */\n", indexes .iter() .map(|i| i.to_string()) @@ -61,12 +60,35 @@ export const VERSION = \"v{}\"; .join(" | ") ); - contents += "\n\n/** @typedef {ReturnType} VecIdToIndexes */"; - contents += "\n/** @typedef {keyof VecIdToIndexes} VecId */\n"; + contents += " +/** @typedef {ReturnType} Indexes */ - contents += "\nexport function createVecIdToIndexes() {\n"; +export function createIndexes() { + return { +"; - contents += " return {\n"; + contents += &indexes + .iter() + .enumerate() + .map(|(i_of_i, i)| { + let lowered = i.to_string().to_lowercase(); + format!(" {lowered}: /** @satisfies {{{i}}} */ ({i_of_i}),",) + }) + .collect::>() + .join("\n"); + + contents += " };\n}\n"; + + contents += " +/** @typedef {ReturnType} VecIdToIndexes +/** @typedef {keyof VecIdToIndexes} VecId */ + +/** + * @returns {Record} + */ +export function createVecIdToIndexes() { + return { +"; self.id_to_index_to_vec() .iter() diff --git a/crates/brk_server/src/api/interface/mod.rs b/crates/brk_server/src/api/interface/mod.rs index cf92afe3c..f6e18b74d 100644 --- a/crates/brk_server/src/api/interface/mod.rs +++ b/crates/brk_server/src/api/interface/mod.rs @@ -4,9 +4,7 @@ use axum::{ http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, }; -use brk_core::DateIndex; use brk_interface::{Format, Output, Params}; -use brk_vec::{CollectableVec, StoredVec}; use color_eyre::eyre::eyre; use crate::traits::{HeaderMapExtended, ResponseExtended}; @@ -50,20 +48,19 @@ fn req_to_response_res( let to = params.to(); let format = params.format(); + // TODO: From and to should be capped here + let weight = vecs .iter() - .map(|(_, v)| { - let len = v.len(); - let count = StoredVec::::range_count(from, to, len); - count * v.value_type_to_size_of() - }) + .map(|(_, v)| v.range_weight(from, to)) .sum::(); if weight > MAX_WEIGHT { return Err(eyre!("Request is too heavy, max weight is {MAX_WEIGHT}")); } - let etag = vecs.first().unwrap().1.etag(to); + // TODO: height should be from vec, but good enough for now + let etag = vecs.first().unwrap().1.etag(interface.get_height(), to); if headers .get_if_none_match() diff --git a/crates/brk_server/src/traits/header_map.rs b/crates/brk_server/src/traits/header_map.rs index 18992c830..63e7e2c0f 100644 --- a/crates/brk_server/src/traits/header_map.rs +++ b/crates/brk_server/src/traits/header_map.rs @@ -8,7 +8,6 @@ use axum::http::{ header::{self, IF_MODIFIED_SINCE, IF_NONE_MATCH}, }; use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone}; -use log::info; const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT"; @@ -140,7 +139,11 @@ impl HeaderMapExtended for HeaderMap { // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types fn insert_content_type(&mut self, path: &Path) { - match path.extension().unwrap().to_str().unwrap() { + match path + .extension() + .map(|s| s.to_str().unwrap_or_default()) + .unwrap_or_default() + { "js" => self.insert_content_type_application_javascript(), "json" | "map" => self.insert_content_type_application_json(), "html" => self.insert_content_type_text_html(), @@ -152,10 +155,7 @@ impl HeaderMapExtended for HeaderMap { "jpg" | "jpeg" => self.insert_content_type_image_jpeg(), "png" => self.insert_content_type_image_png(), "webmanifest" => self.insert_content_type_application_manifest_json(), - extension => { - info!("Extension unsupported: {extension}"); - panic!() - } + _ => {} } } diff --git a/crates/brk_state/Cargo.toml b/crates/brk_state/Cargo.toml deleted file mode 100644 index fc30b22fd..000000000 --- a/crates/brk_state/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "brk_state" -description = "Various states used in the Bitcoin Research Kit" -version.workspace = true -edition.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true - -[dependencies] -bincode = { workspace = true } -brk_core = { workspace = true } -derive_deref = { workspace = true } -serde = { workspace = true } -zerocopy = { workspace = true } -zerocopy-derive = { workspace = true } - -[package.metadata.cargo-machete] -ignored = ["zerocopy"] diff --git a/crates/brk_state/README.md b/crates/brk_state/README.md deleted file mode 100644 index 56e84e077..000000000 --- a/crates/brk_state/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# BRK State - -Various states used in the Bitcoin Research Kit diff --git a/crates/brk_state/src/hot.rs b/crates/brk_state/src/hot.rs deleted file mode 100644 index ce7c8861f..000000000 --- a/crates/brk_state/src/hot.rs +++ /dev/null @@ -1,142 +0,0 @@ -pub struct OneShotSats { - pub price_paid_state: PricePaidState, - pub unrealized_block_state: UnrealizedState, - pub unrealized_date_state: Option, -} - -pub struct UnrealizedState { - supply_in_profit: Sats, - // supply_in_loss: Sats, - unrealized_profit: Dollars, - unrealized_loss: Dollars, -} - -// Why option ? -#[derive(Default, Debug)] -pub struct PricePaidState { - pp_p5: Option, - pp_p10: Option, - pp_p15: Option, - pp_p20: Option, - pp_p25: Option, - pp_p30: Option, - pp_p35: Option, - pp_p40: Option, - pp_p45: Option, - pp_median: Option, - pp_p55: Option, - pp_p60: Option, - pp_p65: Option, - pp_p70: Option, - pp_p75: Option, - pp_p80: Option, - pp_p85: Option, - pp_p90: Option, - pp_p95: Option, - - processed_amount: Sats, -} - -pub struct PricePaidStateFull { - pp_p1: Option, - pp_p2: Option, - pp_p3: Option, - pp_p4: Option, - pp_p5: Option, - pp_p6: Option, - pp_p7: Option, - pp_p8: Option, - pp_p9: Option, - pp_p10: Option, - pp_p11: Option, - pp_p12: Option, - pp_p13: Option, - pp_p14: Option, - pp_p15: Option, - pp_p16: Option, - pp_p17: Option, - pp_p18: Option, - pp_p19: Option, - pp_p20: Option, - pp_p21: Option, - pp_p22: Option, - pp_p23: Option, - pp_p24: Option, - pp_p25: Option, - pp_p26: Option, - pp_p27: Option, - pp_p28: Option, - pp_p29: Option, - pp_p30: Option, - pp_p31: Option, - pp_p32: Option, - pp_p33: Option, - pp_p34: Option, - pp_p35: Option, - pp_p36: Option, - pp_p37: Option, - pp_p38: Option, - pp_p39: Option, - pp_p40: Option, - pp_p41: Option, - pp_p42: Option, - pp_p43: Option, - pp_p44: Option, - pp_p45: Option, - pp_p46: Option, - pp_p47: Option, - pp_p48: Option, - pp_p49: Option, - pp_p50: Option, - pp_p51: Option, - pp_p52: Option, - pp_p53: Option, - pp_p54: Option, - pp_p55: Option, - pp_p56: Option, - pp_p57: Option, - pp_p58: Option, - pp_p59: Option, - pp_p60: Option, - pp_p61: Option, - pp_p62: Option, - pp_p63: Option, - pp_p64: Option, - pp_p65: Option, - pp_p66: Option, - pp_p67: Option, - pp_p68: Option, - pp_p69: Option, - pp_p70: Option, - pp_p71: Option, - pp_p72: Option, - pp_p73: Option, - pp_p74: Option, - pp_p75: Option, - pp_p76: Option, - pp_p77: Option, - pp_p78: Option, - pp_p79: Option, - pp_p80: Option, - pp_p81: Option, - pp_p82: Option, - pp_p83: Option, - pp_p84: Option, - pp_p85: Option, - pp_p86: Option, - pp_p87: Option, - pp_p88: Option, - pp_p89: Option, - pp_p90: Option, - pp_p91: Option, - pp_p92: Option, - pp_p93: Option, - pp_p94: Option, - pp_p95: Option, - pp_p96: Option, - pp_p97: Option, - pp_p98: Option, - pp_p99: Option, - - processed_amount: Sats, -} diff --git a/crates/brk_state/src/lib.rs b/crates/brk_state/src/lib.rs deleted file mode 100644 index 049eca714..000000000 --- a/crates/brk_state/src/lib.rs +++ /dev/null @@ -1,24 +0,0 @@ -#![doc = include_str!("../README.md")] -// #![doc = "\n## Example\n\n```rust"] -// #![doc = include_str!("../examples/main.rs")] -// #![doc = "```"] - -mod block; -mod cohort; -mod outputs; -mod realized; -// mod hot; -mod price_to_amount; -mod supply; -mod transacted; -mod unrealized; - -pub use block::*; -pub use cohort::*; -pub use outputs::*; -pub use realized::*; -pub use unrealized::*; -// pub use hot::*; -pub use price_to_amount::*; -pub use supply::*; -pub use transacted::*; diff --git a/crates/brk_state/src/outputs/by_date_range.rs b/crates/brk_state/src/outputs/by_date_range.rs deleted file mode 100644 index d482171e1..000000000 --- a/crates/brk_state/src/outputs/by_date_range.rs +++ /dev/null @@ -1,126 +0,0 @@ -use super::OutputFilter; - -#[derive(Default, Clone)] -pub struct OutputsByDateRange { - pub start_to_1d: T, - pub _1d_to_1w: T, - pub _1w_to_1m: T, - pub _1m_to_2m: T, - pub _2m_to_3m: T, - pub _3m_to_4m: T, - pub _4m_to_5m: T, - pub _5m_to_6m: T, - pub _6m_to_1y: T, - pub _1y_to_2y: T, - pub _2y_to_3y: T, - pub _3y_to_4y: T, - pub _4y_to_5y: T, - pub _5y_to_6y: T, - pub _6y_to_7y: T, - pub _7y_to_8y: T, - pub _8y_to_10y: T, - pub _10y_to_15y: T, - pub _15y_to_end: T, -} - -impl From> for OutputsByDateRange<(OutputFilter, T)> { - fn from(value: OutputsByDateRange) -> Self { - Self { - start_to_1d: (OutputFilter::To(1), value.start_to_1d), - _1d_to_1w: (OutputFilter::Range(1..7), value._1d_to_1w), - _1w_to_1m: (OutputFilter::Range(7..30), value._1w_to_1m), - _1m_to_2m: (OutputFilter::Range(30..2 * 30), value._1m_to_2m), - _2m_to_3m: (OutputFilter::Range(2 * 30..3 * 30), value._2m_to_3m), - _3m_to_4m: (OutputFilter::Range(3 * 30..4 * 30), value._3m_to_4m), - _4m_to_5m: (OutputFilter::Range(4 * 30..5 * 30), value._4m_to_5m), - _5m_to_6m: (OutputFilter::Range(5 * 30..6 * 30), value._5m_to_6m), - _6m_to_1y: (OutputFilter::Range(6 * 30..365), value._6m_to_1y), - _1y_to_2y: (OutputFilter::Range(365..2 * 365), value._1y_to_2y), - _2y_to_3y: (OutputFilter::Range(2 * 365..3 * 365), value._2y_to_3y), - _3y_to_4y: (OutputFilter::Range(3 * 365..4 * 365), value._3y_to_4y), - _4y_to_5y: (OutputFilter::Range(4 * 365..5 * 365), value._4y_to_5y), - _5y_to_6y: (OutputFilter::Range(5 * 365..6 * 365), value._5y_to_6y), - _6y_to_7y: (OutputFilter::Range(6 * 365..7 * 365), value._6y_to_7y), - _7y_to_8y: (OutputFilter::Range(7 * 365..8 * 365), value._7y_to_8y), - _8y_to_10y: (OutputFilter::Range(8 * 365..10 * 365), value._8y_to_10y), - _10y_to_15y: (OutputFilter::Range(10 * 365..15 * 365), value._10y_to_15y), - _15y_to_end: (OutputFilter::From(15 * 365), value._15y_to_end), - } - } -} - -impl OutputsByDateRange { - pub fn as_vec(&mut self) -> [&T; 19] { - [ - &self.start_to_1d, - &self._1d_to_1w, - &self._1w_to_1m, - &self._1m_to_2m, - &self._2m_to_3m, - &self._3m_to_4m, - &self._4m_to_5m, - &self._5m_to_6m, - &self._6m_to_1y, - &self._1y_to_2y, - &self._2y_to_3y, - &self._3y_to_4y, - &self._4y_to_5y, - &self._5y_to_6y, - &self._6y_to_7y, - &self._7y_to_8y, - &self._8y_to_10y, - &self._10y_to_15y, - &self._15y_to_end, - ] - } - - pub fn as_mut_vec(&mut self) -> [&mut T; 19] { - [ - &mut self.start_to_1d, - &mut self._1d_to_1w, - &mut self._1w_to_1m, - &mut self._1m_to_2m, - &mut self._2m_to_3m, - &mut self._3m_to_4m, - &mut self._4m_to_5m, - &mut self._5m_to_6m, - &mut self._6m_to_1y, - &mut self._1y_to_2y, - &mut self._2y_to_3y, - &mut self._3y_to_4y, - &mut self._4y_to_5y, - &mut self._5y_to_6y, - &mut self._6y_to_7y, - &mut self._7y_to_8y, - &mut self._8y_to_10y, - &mut self._10y_to_15y, - &mut self._15y_to_end, - ] - } -} - -impl OutputsByDateRange<(OutputFilter, T)> { - pub fn vecs(&self) -> [&T; 19] { - [ - &self.start_to_1d.1, - &self._1d_to_1w.1, - &self._1w_to_1m.1, - &self._1m_to_2m.1, - &self._2m_to_3m.1, - &self._3m_to_4m.1, - &self._4m_to_5m.1, - &self._5m_to_6m.1, - &self._6m_to_1y.1, - &self._1y_to_2y.1, - &self._2y_to_3y.1, - &self._3y_to_4y.1, - &self._4y_to_5y.1, - &self._5y_to_6y.1, - &self._6y_to_7y.1, - &self._7y_to_8y.1, - &self._8y_to_10y.1, - &self._10y_to_15y.1, - &self._15y_to_end.1, - ] - } -} diff --git a/crates/brk_state/src/outputs/by_from_date.rs b/crates/brk_state/src/outputs/by_from_date.rs deleted file mode 100644 index e590c4ead..000000000 --- a/crates/brk_state/src/outputs/by_from_date.rs +++ /dev/null @@ -1,98 +0,0 @@ -use super::OutputFilter; - -#[derive(Default, Clone)] -pub struct OutputsByFromDate { - pub _1d: T, - pub _1w: T, - pub _1m: T, - pub _2m: T, - pub _3m: T, - pub _4m: T, - pub _5m: T, - pub _6m: T, - pub _1y: T, - pub _2y: T, - pub _3y: T, - pub _4y: T, - pub _5y: T, - pub _6y: T, - pub _7y: T, - pub _8y: T, - pub _10y: T, - pub _15y: T, -} - -impl OutputsByFromDate { - pub fn as_mut_vec(&mut self) -> [&mut T; 18] { - [ - &mut self._1d, - &mut self._1w, - &mut self._1m, - &mut self._2m, - &mut self._3m, - &mut self._4m, - &mut self._5m, - &mut self._6m, - &mut self._1y, - &mut self._2y, - &mut self._3y, - &mut self._4y, - &mut self._5y, - &mut self._6y, - &mut self._7y, - &mut self._8y, - &mut self._10y, - &mut self._15y, - ] - } -} - -impl OutputsByFromDate<(OutputFilter, T)> { - pub fn vecs(&self) -> [&T; 18] { - [ - &self._1d.1, - &self._1w.1, - &self._1m.1, - &self._2m.1, - &self._3m.1, - &self._4m.1, - &self._5m.1, - &self._6m.1, - &self._1y.1, - &self._2y.1, - &self._3y.1, - &self._4y.1, - &self._5y.1, - &self._6y.1, - &self._7y.1, - &self._8y.1, - &self._10y.1, - &self._15y.1, - ] - } -} - -impl From> for OutputsByFromDate<(OutputFilter, T)> { - fn from(value: OutputsByFromDate) -> Self { - Self { - _1d: (OutputFilter::From(1), value._1d), - _1w: (OutputFilter::From(7), value._1w), - _1m: (OutputFilter::From(30), value._1m), - _2m: (OutputFilter::From(2 * 30), value._2m), - _3m: (OutputFilter::From(3 * 30), value._3m), - _4m: (OutputFilter::From(4 * 30), value._4m), - _5m: (OutputFilter::From(5 * 30), value._5m), - _6m: (OutputFilter::From(6 * 30), value._6m), - _1y: (OutputFilter::From(365), value._1y), - _2y: (OutputFilter::From(2 * 365), value._2y), - _3y: (OutputFilter::From(3 * 365), value._3y), - _4y: (OutputFilter::From(4 * 365), value._4y), - _5y: (OutputFilter::From(5 * 365), value._5y), - _6y: (OutputFilter::From(6 * 365), value._6y), - _7y: (OutputFilter::From(7 * 365), value._7y), - _8y: (OutputFilter::From(8 * 365), value._8y), - _10y: (OutputFilter::From(10 * 365), value._10y), - _15y: (OutputFilter::From(15 * 365), value._15y), - } - } -} diff --git a/crates/brk_state/src/outputs/by_from_size.rs b/crates/brk_state/src/outputs/by_from_size.rs deleted file mode 100644 index 27d980d85..000000000 --- a/crates/brk_state/src/outputs/by_from_size.rs +++ /dev/null @@ -1,50 +0,0 @@ -use brk_core::Sats; - -use super::OutputFilter; - -#[derive(Default, Clone)] -pub struct OutputsByFromSize { - pub _1_000sats: T, - pub _1btc: T, - pub _10btc: T, - pub _100btc: T, -} - -impl OutputsByFromSize { - pub fn as_mut_vec(&mut self) -> [&mut T; 4] { - [ - &mut self._1_000sats, - &mut self._1btc, - &mut self._10btc, - &mut self._100btc, - ] - } -} - -impl OutputsByFromSize<(OutputFilter, T)> { - pub fn vecs(&self) -> [&T; 4] { - [ - &self._1_000sats.1, - &self._1btc.1, - &self._10btc.1, - &self._100btc.1, - ] - } -} - -impl From> for OutputsByFromSize<(OutputFilter, T)> { - fn from(value: OutputsByFromSize) -> Self { - Self { - _1_000sats: (OutputFilter::From(1_000), value._1_000sats), - _1btc: (OutputFilter::From(usize::from(Sats::ONE_BTC)), value._1btc), - _10btc: ( - OutputFilter::From(usize::from(10 * Sats::ONE_BTC)), - value._10btc, - ), - _100btc: ( - OutputFilter::From(usize::from(100 * Sats::ONE_BTC)), - value._100btc, - ), - } - } -} diff --git a/crates/brk_state/src/outputs/by_size_range.rs b/crates/brk_state/src/outputs/by_size_range.rs deleted file mode 100644 index 0d376acf9..000000000 --- a/crates/brk_state/src/outputs/by_size_range.rs +++ /dev/null @@ -1,178 +0,0 @@ -use super::OutputFilter; - -#[derive(Default, Clone)] -pub struct OutputsBySizeRange { - pub _0sats: T, - pub from_1sat_to_10sats: T, - pub from_10sats_to_100sats: T, - pub from_100sats_to_1_000sats: T, - pub from_1_000sats_to_10_000sats: T, - pub from_10_000sats_to_100_000sats: T, - pub from_100_000sats_to_1_000_000sats: T, - pub from_1_000_000sats_to_10_000_000sats: T, - pub from_10_000_000sats_to_1btc: T, - pub from_1btc_to_10btc: T, - pub from_10btc_to_100btc: T, - pub from_100btc_to_1_000btc: T, - pub from_1_000btc_to_10_000btc: T, - pub from_10_000btc_to_100_000btc: T, - pub from_100_000btc: T, -} - -impl From> for OutputsBySizeRange<(OutputFilter, T)> { - fn from(value: OutputsBySizeRange) -> Self { - #[allow(clippy::inconsistent_digit_grouping)] - Self { - _0sats: (OutputFilter::To(1), value._0sats), - from_1sat_to_10sats: (OutputFilter::Range(1..10), value.from_1sat_to_10sats), - from_10sats_to_100sats: (OutputFilter::Range(10..100), value.from_10sats_to_100sats), - from_100sats_to_1_000sats: ( - OutputFilter::Range(100..1_000), - value.from_100sats_to_1_000sats, - ), - from_1_000sats_to_10_000sats: ( - OutputFilter::Range(1_000..10_000), - value.from_1_000sats_to_10_000sats, - ), - from_10_000sats_to_100_000sats: ( - OutputFilter::Range(10_000..100_000), - value.from_10_000sats_to_100_000sats, - ), - from_100_000sats_to_1_000_000sats: ( - OutputFilter::Range(100_000..1_000_000), - value.from_100_000sats_to_1_000_000sats, - ), - from_1_000_000sats_to_10_000_000sats: ( - OutputFilter::Range(1_000_000..10_000_000), - value.from_1_000_000sats_to_10_000_000sats, - ), - from_10_000_000sats_to_1btc: ( - OutputFilter::Range(10_000_000..1_00_000_000), - value.from_10_000_000sats_to_1btc, - ), - from_1btc_to_10btc: ( - OutputFilter::Range(1_00_000_000..10_00_000_000), - value.from_1btc_to_10btc, - ), - from_10btc_to_100btc: ( - OutputFilter::Range(10_00_000_000..100_00_000_000), - value.from_10btc_to_100btc, - ), - from_100btc_to_1_000btc: ( - OutputFilter::Range(100_00_000_000..1_000_00_000_000), - value.from_100btc_to_1_000btc, - ), - from_1_000btc_to_10_000btc: ( - OutputFilter::Range(1_000_00_000_000..10_000_00_000_000), - value.from_1_000btc_to_10_000btc, - ), - from_10_000btc_to_100_000btc: ( - OutputFilter::Range(10_000_00_000_000..100_000_00_000_000), - value.from_10_000btc_to_100_000btc, - ), - from_100_000btc: ( - OutputFilter::From(100_000_00_000_000), - value.from_100_000btc, - ), - } - } -} - -impl OutputsBySizeRange { - #[allow(clippy::inconsistent_digit_grouping)] - pub fn get_mut(&mut self, group: usize) -> &mut T { - if group == 0 { - &mut self._0sats - } else if group == 1 { - &mut self.from_1sat_to_10sats - } else if group == 10 { - &mut self.from_10sats_to_100sats - } else if group == 100 { - &mut self.from_100sats_to_1_000sats - } else if group == 1_000 { - &mut self.from_1_000sats_to_10_000sats - } else if group == 10_000 { - &mut self.from_10_000sats_to_100_000sats - } else if group == 100_000 { - &mut self.from_100_000sats_to_1_000_000sats - } else if group == 1_000_000 { - &mut self.from_1_000_000sats_to_10_000_000sats - } else if group == 10_000_000 { - &mut self.from_10_000_000sats_to_1btc - } else if group == 1_00_000_000 { - &mut self.from_1btc_to_10btc - } else if group == 10_00_000_000 { - &mut self.from_10btc_to_100btc - } else if group == 100_00_000_000 { - &mut self.from_100btc_to_1_000btc - } else if group == 1_000_00_000_000 { - &mut self.from_1_000btc_to_10_000btc - } else if group == 10_000_00_000_000 { - &mut self.from_10_000btc_to_100_000btc - } else { - &mut self.from_100_000btc - } - } - - pub fn as_vec(&self) -> [&T; 15] { - [ - &self._0sats, - &self.from_1sat_to_10sats, - &self.from_10sats_to_100sats, - &self.from_100sats_to_1_000sats, - &self.from_1_000sats_to_10_000sats, - &self.from_10_000sats_to_100_000sats, - &self.from_100_000sats_to_1_000_000sats, - &self.from_1_000_000sats_to_10_000_000sats, - &self.from_10_000_000sats_to_1btc, - &self.from_1btc_to_10btc, - &self.from_10btc_to_100btc, - &self.from_100btc_to_1_000btc, - &self.from_1_000btc_to_10_000btc, - &self.from_10_000btc_to_100_000btc, - &self.from_100_000btc, - ] - } - - pub fn as_mut_vec(&mut self) -> [&mut T; 15] { - [ - &mut self._0sats, - &mut self.from_1sat_to_10sats, - &mut self.from_10sats_to_100sats, - &mut self.from_100sats_to_1_000sats, - &mut self.from_1_000sats_to_10_000sats, - &mut self.from_10_000sats_to_100_000sats, - &mut self.from_100_000sats_to_1_000_000sats, - &mut self.from_1_000_000sats_to_10_000_000sats, - &mut self.from_10_000_000sats_to_1btc, - &mut self.from_1btc_to_10btc, - &mut self.from_10btc_to_100btc, - &mut self.from_100btc_to_1_000btc, - &mut self.from_1_000btc_to_10_000btc, - &mut self.from_10_000btc_to_100_000btc, - &mut self.from_100_000btc, - ] - } -} - -impl OutputsBySizeRange<(OutputFilter, T)> { - pub fn vecs(&self) -> [&T; 15] { - [ - &self._0sats.1, - &self.from_1sat_to_10sats.1, - &self.from_10sats_to_100sats.1, - &self.from_100sats_to_1_000sats.1, - &self.from_1_000sats_to_10_000sats.1, - &self.from_10_000sats_to_100_000sats.1, - &self.from_100_000sats_to_1_000_000sats.1, - &self.from_1_000_000sats_to_10_000_000sats.1, - &self.from_10_000_000sats_to_1btc.1, - &self.from_1btc_to_10btc.1, - &self.from_10btc_to_100btc.1, - &self.from_100btc_to_1_000btc.1, - &self.from_1_000btc_to_10_000btc.1, - &self.from_10_000btc_to_100_000btc.1, - &self.from_100_000btc.1, - ] - } -} diff --git a/crates/brk_state/src/outputs/by_term.rs b/crates/brk_state/src/outputs/by_term.rs deleted file mode 100644 index 1a80fbb34..000000000 --- a/crates/brk_state/src/outputs/by_term.rs +++ /dev/null @@ -1,28 +0,0 @@ -use super::OutputFilter; - -#[derive(Default, Clone)] -pub struct OutputsByTerm { - pub short: T, - pub long: T, -} - -impl OutputsByTerm { - pub fn as_mut_vec(&mut self) -> [&mut T; 2] { - [&mut self.short, &mut self.long] - } -} - -impl OutputsByTerm<(OutputFilter, T)> { - pub fn vecs(&self) -> [&T; 2] { - [&self.short.1, &self.long.1] - } -} - -impl From> for OutputsByTerm<(OutputFilter, T)> { - fn from(value: OutputsByTerm) -> Self { - Self { - short: (OutputFilter::To(5 * 30), value.short), - long: (OutputFilter::From(5 * 30), value.long), - } - } -} diff --git a/crates/brk_state/src/outputs/by_up_to_date.rs b/crates/brk_state/src/outputs/by_up_to_date.rs deleted file mode 100644 index e28ecd0bb..000000000 --- a/crates/brk_state/src/outputs/by_up_to_date.rs +++ /dev/null @@ -1,98 +0,0 @@ -use super::OutputFilter; - -#[derive(Default, Clone)] -pub struct OutputsByUpToDate { - pub _1d: T, - pub _1w: T, - pub _1m: T, - pub _2m: T, - pub _3m: T, - pub _4m: T, - pub _5m: T, - pub _6m: T, - pub _1y: T, - pub _2y: T, - pub _3y: T, - pub _4y: T, - pub _5y: T, - pub _6y: T, - pub _7y: T, - pub _8y: T, - pub _10y: T, - pub _15y: T, -} - -impl OutputsByUpToDate { - pub fn as_mut_vec(&mut self) -> [&mut T; 18] { - [ - &mut self._1d, - &mut self._1w, - &mut self._1m, - &mut self._2m, - &mut self._3m, - &mut self._4m, - &mut self._5m, - &mut self._6m, - &mut self._1y, - &mut self._2y, - &mut self._3y, - &mut self._4y, - &mut self._5y, - &mut self._6y, - &mut self._7y, - &mut self._8y, - &mut self._10y, - &mut self._15y, - ] - } -} - -impl OutputsByUpToDate<(OutputFilter, T)> { - pub fn vecs(&self) -> [&T; 18] { - [ - &self._1d.1, - &self._1w.1, - &self._1m.1, - &self._2m.1, - &self._3m.1, - &self._4m.1, - &self._5m.1, - &self._6m.1, - &self._1y.1, - &self._2y.1, - &self._3y.1, - &self._4y.1, - &self._5y.1, - &self._6y.1, - &self._7y.1, - &self._8y.1, - &self._10y.1, - &self._15y.1, - ] - } -} - -impl From> for OutputsByUpToDate<(OutputFilter, T)> { - fn from(value: OutputsByUpToDate) -> Self { - Self { - _1d: (OutputFilter::To(1), value._1d), - _1w: (OutputFilter::To(7), value._1w), - _1m: (OutputFilter::To(30), value._1m), - _2m: (OutputFilter::To(2 * 30), value._2m), - _3m: (OutputFilter::To(3 * 30), value._3m), - _4m: (OutputFilter::To(4 * 30), value._4m), - _5m: (OutputFilter::To(5 * 30), value._5m), - _6m: (OutputFilter::To(6 * 30), value._6m), - _1y: (OutputFilter::To(365), value._1y), - _2y: (OutputFilter::To(2 * 365), value._2y), - _3y: (OutputFilter::To(3 * 365), value._3y), - _4y: (OutputFilter::To(4 * 365), value._4y), - _5y: (OutputFilter::To(5 * 365), value._5y), - _6y: (OutputFilter::To(6 * 365), value._6y), - _7y: (OutputFilter::To(7 * 365), value._7y), - _8y: (OutputFilter::To(8 * 365), value._8y), - _10y: (OutputFilter::To(10 * 365), value._10y), - _15y: (OutputFilter::To(15 * 365), value._15y), - } - } -} diff --git a/crates/brk_state/src/outputs/by_up_to_size.rs b/crates/brk_state/src/outputs/by_up_to_size.rs deleted file mode 100644 index 5987ee28f..000000000 --- a/crates/brk_state/src/outputs/by_up_to_size.rs +++ /dev/null @@ -1,54 +0,0 @@ -use brk_core::Sats; - -use super::OutputFilter; - -#[derive(Default, Clone)] -pub struct OutputsByUpToSize { - pub _1_000sats: T, - pub _10_000sats: T, - pub _1btc: T, - pub _10btc: T, - pub _100btc: T, -} - -impl OutputsByUpToSize { - pub fn as_mut_vec(&mut self) -> [&mut T; 5] { - [ - &mut self._1_000sats, - &mut self._10_000sats, - &mut self._1btc, - &mut self._10btc, - &mut self._100btc, - ] - } -} - -impl OutputsByUpToSize<(OutputFilter, T)> { - pub fn vecs(&self) -> [&T; 5] { - [ - &self._1_000sats.1, - &self._10_000sats.1, - &self._1btc.1, - &self._10btc.1, - &self._100btc.1, - ] - } -} - -impl From> for OutputsByUpToSize<(OutputFilter, T)> { - fn from(value: OutputsByUpToSize) -> Self { - Self { - _1_000sats: (OutputFilter::To(1_000), value._1_000sats), - _10_000sats: (OutputFilter::To(10_000), value._10_000sats), - _1btc: (OutputFilter::To(usize::from(Sats::ONE_BTC)), value._1btc), - _10btc: ( - OutputFilter::To(usize::from(10 * Sats::ONE_BTC)), - value._10btc, - ), - _100btc: ( - OutputFilter::To(usize::from(100 * Sats::ONE_BTC)), - value._100btc, - ), - } - } -} diff --git a/crates/brk_state/src/outputs/filter.rs b/crates/brk_state/src/outputs/filter.rs deleted file mode 100644 index 613258280..000000000 --- a/crates/brk_state/src/outputs/filter.rs +++ /dev/null @@ -1,51 +0,0 @@ -use std::ops::Range; - -use brk_core::{HalvingEpoch, OutputType}; - -#[derive(Debug, Clone)] -pub enum OutputFilter { - All, - To(usize), - Range(Range), - From(usize), - Epoch(HalvingEpoch), - Type(OutputType), -} - -impl OutputFilter { - pub fn contains(&self, value: usize) -> bool { - match self { - OutputFilter::All => true, - OutputFilter::To(to) => *to > value, - OutputFilter::From(from) => *from <= value, - OutputFilter::Range(r) => r.contains(&value), - OutputFilter::Epoch(_) => false, - OutputFilter::Type(_) => false, - } - } - - pub fn includes(&self, other: &OutputFilter) -> bool { - match self { - OutputFilter::All => true, - OutputFilter::To(to) => match other { - OutputFilter::All => false, - OutputFilter::To(to2) => to >= to2, - OutputFilter::Range(range) => range.end <= *to, - OutputFilter::From(_) => false, - OutputFilter::Epoch(_) => false, - OutputFilter::Type(_) => false, - }, - OutputFilter::From(from) => match other { - OutputFilter::All => false, - OutputFilter::To(_) => false, - OutputFilter::Range(range) => range.start >= *from, - OutputFilter::From(from2) => from <= from2, - OutputFilter::Epoch(_) => false, - OutputFilter::Type(_) => false, - }, - OutputFilter::Range(_) => false, - OutputFilter::Epoch(_) => false, - OutputFilter::Type(_) => false, - } - } -} diff --git a/crates/brk_state/src/outputs/mod.rs b/crates/brk_state/src/outputs/mod.rs deleted file mode 100644 index 97efb4c87..000000000 --- a/crates/brk_state/src/outputs/mod.rs +++ /dev/null @@ -1,120 +0,0 @@ -mod by_date_range; -mod by_epoch; -mod by_from_date; -mod by_from_size; -mod by_size_range; -mod by_spendable_type; -mod by_term; -mod by_type; -mod by_unspendable_type; -mod by_up_to_date; -mod by_up_to_size; -// mod by_value; -mod filter; - -pub use by_date_range::*; -pub use by_epoch::*; -pub use by_from_date::*; -pub use by_from_size::*; -pub use by_size_range::*; -pub use by_spendable_type::*; -pub use by_term::*; -pub use by_type::*; -pub use by_unspendable_type::*; -pub use by_up_to_date::*; -pub use by_up_to_size::*; -// pub use by_value::*; -pub use filter::*; - -#[derive(Default, Clone)] -pub struct Outputs { - pub all: T, - pub by_date_range: OutputsByDateRange, - pub by_epoch: OutputsByEpoch, - pub by_from_date: OutputsByFromDate, - pub by_from_size: OutputsByFromSize, - pub by_size_range: OutputsBySizeRange, - pub by_term: OutputsByTerm, - pub by_type: OutputsBySpendableType, - pub by_up_to_date: OutputsByUpToDate, - pub by_up_to_size: OutputsByUpToSize, - // Needs whole UTXO set, TODO later - // pub by_value: OutputsByValue, -} - -impl Outputs { - pub fn as_mut_vecs(&mut self) -> Vec<&mut T> { - [&mut self.all] - .into_iter() - .chain(self.by_term.as_mut_vec()) - .chain(self.by_up_to_date.as_mut_vec()) - .chain(self.by_from_date.as_mut_vec()) - .chain(self.by_from_size.as_mut_vec()) - .chain(self.by_date_range.as_mut_vec()) - .chain(self.by_epoch.as_mut_vec()) - .chain(self.by_size_range.as_mut_vec()) - .chain(self.by_up_to_size.as_mut_vec()) - .chain(self.by_type.as_mut_vec()) - // .chain(self.by_value.as_mut_vec()) - .collect::>() - } - - pub fn as_mut_separate_vecs(&mut self) -> Vec<&mut T> { - self.by_date_range - .as_mut_vec() - .into_iter() - .chain(self.by_epoch.as_mut_vec()) - .chain(self.by_size_range.as_mut_vec()) - .chain(self.by_type.as_mut_vec()) - // .chain(self.by_value.as_mut_vec()) - .collect::>() - } - - pub fn as_mut_overlapping_vecs(&mut self) -> Vec<&mut T> { - [&mut self.all] - .into_iter() - .chain(self.by_term.as_mut_vec()) - .chain(self.by_up_to_date.as_mut_vec()) - .chain(self.by_from_date.as_mut_vec()) - .chain(self.by_up_to_size.as_mut_vec()) - .chain(self.by_from_size.as_mut_vec()) - .collect::>() - } -} - -impl Outputs<(OutputFilter, T)> { - pub fn vecs(&self) -> Vec<&T> { - [&self.all.1] - .into_iter() - .chain(self.by_term.vecs()) - .chain(self.by_up_to_date.vecs()) - .chain(self.by_from_date.vecs()) - .chain(self.by_date_range.vecs()) - .chain(self.by_epoch.vecs()) - .chain(self.by_size_range.vecs()) - .chain(self.by_type.vecs()) - .chain(self.by_up_to_size.vecs()) - .chain(self.by_from_size.vecs()) - // .chain(self.by_value.vecs()) - .collect::>() - } -} - -impl From> for Outputs<(OutputFilter, T)> { - fn from(value: Outputs) -> Self { - Self { - all: (OutputFilter::All, value.all), - by_term: OutputsByTerm::from(value.by_term), - by_up_to_date: OutputsByUpToDate::from(value.by_up_to_date), - by_from_date: OutputsByFromDate::from(value.by_from_date), - by_date_range: OutputsByDateRange::from(value.by_date_range), - by_epoch: OutputsByEpoch::from(value.by_epoch), - by_size_range: OutputsBySizeRange::from(value.by_size_range), - by_up_to_size: OutputsByUpToSize::from(value.by_up_to_size), - by_from_size: OutputsByFromSize::from(value.by_from_size), - // Needs whole UTXO set, TODO later - // by_value: OutputsByValue, - by_type: OutputsBySpendableType::from(value.by_type), - } - } -} diff --git a/crates/brk_state/src/transacted.rs b/crates/brk_state/src/transacted.rs deleted file mode 100644 index 271892c5a..000000000 --- a/crates/brk_state/src/transacted.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::{ - collections::BTreeMap, - mem, - ops::{Add, AddAssign}, -}; - -use brk_core::{OutputType, Sats}; - -use super::{OutputsByType, SupplyState}; - -#[derive(Default, Debug)] -pub struct Transacted { - pub spendable_supply: SupplyState, - pub by_type: OutputsByType, - pub by_size_group: BTreeMap, -} - -impl Transacted { - #[allow(clippy::inconsistent_digit_grouping)] - pub fn iterate(&mut self, value: Sats, _type: OutputType) { - let supply = SupplyState { utxos: 1, value }; - - *self.by_type.get_mut(_type) += &supply; - - if _type.is_unspendable() { - return; - } - - self.spendable_supply += &supply; - - let _value = usize::from(value); - - // Need to be in sync with by_size !! but plenty fast (I think) - if _value == 0 { - *self.by_size_group.entry(0).or_default() += &supply; - } else if _value < 10 { - *self.by_size_group.entry(1).or_default() += &supply; - } else if _value < 100 { - *self.by_size_group.entry(10).or_default() += &supply; - } else if _value < 1_000 { - *self.by_size_group.entry(100).or_default() += &supply; - } else if _value < 10_000 { - *self.by_size_group.entry(1_000).or_default() += &supply; - } else if _value < 100_000 { - *self.by_size_group.entry(10_000).or_default() += &supply; - } else if _value < 1_000_000 { - *self.by_size_group.entry(100_000).or_default() += &supply; - } else if _value < 10_000_000 { - *self.by_size_group.entry(1_000_000).or_default() += &supply; - } else if _value < 1_00_000_000 { - *self.by_size_group.entry(10_000_000).or_default() += &supply; - } else if _value < 10_00_000_000 { - *self.by_size_group.entry(1_00_000_000).or_default() += &supply; - } else if _value < 100_00_000_000 { - *self.by_size_group.entry(10_00_000_000).or_default() += &supply; - } else if _value < 1_000_00_000_000 { - *self.by_size_group.entry(100_00_000_000).or_default() += &supply; - } else if _value < 10_000_00_000_000 { - *self.by_size_group.entry(1_000_00_000_000).or_default() += &supply; - } else if _value < 100_000_00_000_000 { - *self.by_size_group.entry(10_000_00_000_000).or_default() += &supply; - } else { - *self.by_size_group.entry(100_000_00_000_000).or_default() += &supply; - } - } - - fn merge_by_size( - first: BTreeMap, - second: BTreeMap, - ) -> BTreeMap { - let (mut source, to_consume) = if first.len() > second.len() { - (first, second) - } else { - (second, first) - }; - to_consume.into_iter().for_each(|(k, v)| { - *source.entry(k).or_default() += &v; - }); - source - } -} - -impl Add for Transacted { - type Output = Self; - fn add(self, rhs: Self) -> Self::Output { - Self { - spendable_supply: self.spendable_supply + rhs.spendable_supply, - by_type: self.by_type + rhs.by_type, - by_size_group: Self::merge_by_size(self.by_size_group, rhs.by_size_group), - } - } -} - -impl AddAssign for Transacted { - fn add_assign(&mut self, rhs: Self) { - self.by_size_group = - Self::merge_by_size(mem::take(&mut self.by_size_group), rhs.by_size_group); - self.spendable_supply += &rhs.spendable_supply; - self.by_type += rhs.by_type; - } -} diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index 687831ee4..0c1e88c30 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -13,3 +13,4 @@ repository.workspace = true brk_core = { workspace = true } byteview = { workspace = true } fjall = { workspace = true } +log = { workspace = true } diff --git a/crates/brk_store/examples/main.rs b/crates/brk_store/examples/main.rs index d7f80d31d..82400ecdd 100644 --- a/crates/brk_store/examples/main.rs +++ b/crates/brk_store/examples/main.rs @@ -1,7 +1,7 @@ use std::path::Path; use brk_core::{Dollars, Height, Result, Sats, Version}; -use brk_store::Store; +use brk_store::{AnyStore, Store}; fn main() -> Result<()> { let p = Path::new("./examples/_fjall"); @@ -15,5 +15,7 @@ fn main() -> Result<()> { store.commit(Height::ZERO)?; + store.persist()?; + Ok(()) } diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 7b99eace6..2f51e5d51 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -4,13 +4,14 @@ #![doc = "```"] use std::{ + borrow::Cow, collections::{BTreeMap, BTreeSet}, fmt::Debug, - mem, + fs, mem, path::Path, }; -use brk_core::{Height, Result, Value, Version}; +use brk_core::{Height, Result, Version}; use byteview::ByteView; use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace, @@ -18,22 +19,25 @@ use fjall::{ }; mod meta; +mod r#trait; + +use log::info; use meta::*; +pub use r#trait::*; pub struct Store { meta: StoreMeta, - name: String, + name: &'static str, keyspace: TransactionalKeyspace, + // Arc it partition: Option, rtx: ReadTransaction, puts: BTreeMap, dels: BTreeSet, - bloom_filter_bits: Option>, + bloom_filters: Option, } -/// Use default if will read -const DEFAULT_BLOOM_FILTER_BITS: Option = Some(5); -// const CHECK_COLLISISONS: bool = true; +// const CHECK_COLLISIONS: bool = true; const MAJOR_FJALL_VERSION: Version = Version::TWO; pub fn open_keyspace(path: &Path) -> fjall::Result { @@ -53,14 +57,16 @@ where path: &Path, name: &str, version: Version, - bloom_filter_bits: Option>, + bloom_filters: Option, ) -> Result { + fs::create_dir_all(path)?; + let (meta, partition) = StoreMeta::checked_open( keyspace, &path.join(format!("meta/{name}")), MAJOR_FJALL_VERSION + version, || { - Self::open_partition_handle(keyspace, name, bloom_filter_bits).inspect_err(|e| { + Self::open_partition_handle(keyspace, name, bloom_filters).inspect_err(|e| { eprintln!("{e}"); eprintln!("Delete {path:?} and try again"); }) @@ -71,29 +77,35 @@ where Ok(Self { meta, - name: name.to_owned(), + name: Box::leak(Box::new(name.to_string())), keyspace: keyspace.clone(), partition: Some(partition), rtx, puts: BTreeMap::new(), dels: BTreeSet::new(), - bloom_filter_bits, + bloom_filters, }) } - pub fn get(&self, key: &'a K) -> Result>> { + pub fn get(&self, key: &'a K) -> Result>> { if let Some(v) = self.puts.get(key) { - Ok(Some(Value::Ref(v))) + Ok(Some(Cow::Borrowed(v))) } else if let Some(slice) = self .rtx .get(self.partition.as_ref().unwrap(), ByteView::from(key))? { - Ok(Some(Value::Owned(V::from(ByteView::from(slice))))) + Ok(Some(Cow::Owned(V::from(ByteView::from(slice))))) } else { Ok(None) } } + pub fn is_empty(&self) -> Result { + self.rtx + .is_empty(self.partition.as_ref().unwrap()) + .map_err(|e| e.into()) + } + // pub fn puts_first_key_value(&self) -> Option<(&K, &V)> { // self.puts.first_key_value() // } @@ -126,17 +138,17 @@ where pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) { if self.needs(height) { if !self.dels.is_empty() { - // self.dels.remove(&key); - unreachable!("Shouldn't reach this"); + self.dels.remove(&key); + // unreachable!("Shouldn't reach this"); } self.puts.insert(key, value); } } pub fn remove(&mut self, key: K) { - if self.is_empty() { - return; - } + // if self.is_empty()? { + // return Ok(()); + // } if !self.puts.is_empty() { unreachable!("Shouldn't reach this"); @@ -146,6 +158,8 @@ where dbg!(key, &self.meta.path()); unreachable!(); } + + // Ok(()) } // pub fn retain_or_del(&mut self, retain: F) @@ -161,38 +175,55 @@ where // }); // } - pub fn commit(&mut self, height: Height) -> Result<()> { - if self.has(height) && self.puts.is_empty() && self.dels.is_empty() { + fn open_partition_handle( + keyspace: &TransactionalKeyspace, + name: &str, + bloom_filters: Option, + ) -> Result { + let mut options = PartitionCreateOptions::default() + .max_memtable_size(8 * 1024 * 1024) + .manual_journal_persist(true); + + if bloom_filters.is_some_and(|b| !b) { + options = options.bloom_filter_bits(None); + } + + keyspace.open_partition(name, options).map_err(|e| e.into()) + } + + pub fn commit_( + &mut self, + height: Height, + remove: impl Iterator, + insert: impl Iterator, + ) -> Result<()> { + if self.has(height) { return Ok(()); } - self.meta.export(self.len(), height)?; + self.meta.export(height)?; let mut wtx = self.keyspace.write_tx(); let partition = self.partition.as_ref().unwrap(); - mem::take(&mut self.dels) - .into_iter() - .for_each(|key| wtx.remove(partition, ByteView::from(key))); + remove.for_each(|key| wtx.remove(partition, ByteView::from(key))); - mem::take(&mut self.puts) - .into_iter() - .for_each(|(key, value)| { - // if CHECK_COLLISISONS { - // #[allow(unused_must_use)] - // if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) { - // dbg!( - // &key, - // V::try_from(value.as_bytes().into()).unwrap(), - // &self.meta, - // self.rtx.get(&self.partition, key.as_bytes()) - // ); - // unreachable!(); - // } - // } - wtx.insert(partition, ByteView::from(key), ByteView::from(value)) - }); + insert.for_each(|(key, value)| { + // if CHECK_COLLISIONS { + // #[allow(unused_must_use)] + // if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) { + // dbg!( + // &key, + // V::try_from(value.as_bytes().into()).unwrap(), + // &self.meta, + // self.rtx.get(&self.partition, key.as_bytes()) + // ); + // unreachable!(); + // } + // } + wtx.insert(partition, ByteView::from(key), ByteView::from(value)) + }); wtx.commit()?; @@ -200,72 +231,66 @@ where Ok(()) } +} - pub fn rotate_memtable(&self) { - let _ = self.partition.as_ref().unwrap().inner().rotate_memtable(); - } - - pub fn height(&self) -> Option { - self.meta.height() - } - - pub fn len(&self) -> usize { - let len = self.meta.len() + self.puts.len() - self.dels.len(); - if len > 18440000000000000000 { - dbg!(( - len, - self.meta.path(), - self.meta.len(), - self.puts.len(), - &self.dels, - )); - unreachable!() +impl<'a, K, V> AnyStore for Store +where + K: Debug + Clone + From + Ord + 'a, + V: Debug + Clone + From, + ByteView: From + From<&'a K> + From, +{ + fn commit(&mut self, height: Height) -> Result<()> { + if self.puts.is_empty() && self.dels.is_empty() { + self.meta.export(height)?; + return Ok(()); } - len - } - pub fn is_empty(&self) -> bool { - self.len() == 0 + + let dels = mem::take(&mut self.dels); + let puts = mem::take(&mut self.puts); + + self.commit_(height, dels.into_iter(), puts.into_iter()) } - pub fn has(&self, height: Height) -> bool { - self.meta.has(height) - } - pub fn needs(&self, height: Height) -> bool { - self.meta.needs(height) - } - - fn open_partition_handle( - keyspace: &TransactionalKeyspace, - name: &str, - bloom_filter_bits: Option>, - ) -> Result { - keyspace - .open_partition( - name, - PartitionCreateOptions::default() - .bloom_filter_bits(bloom_filter_bits.unwrap_or(DEFAULT_BLOOM_FILTER_BITS)) - .max_memtable_size(8 * 1024 * 1024) - .manual_journal_persist(true), - ) + fn persist(&self) -> Result<()> { + self.keyspace + .persist(PersistMode::SyncAll) .map_err(|e| e.into()) } - pub fn reset_partition(&mut self) -> Result<()> { + fn name(&self) -> &'static str { + self.name + } + + fn reset(&mut self) -> Result<()> { + info!("Resetting {}...", self.name); + let partition: TransactionalPartitionHandle = self.partition.take().unwrap(); self.keyspace.delete_partition(partition)?; - self.keyspace.persist(PersistMode::SyncAll)?; - self.meta.reset(); - let partition = - Self::open_partition_handle(&self.keyspace, &self.name, self.bloom_filter_bits)?; + let partition = Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filters)?; self.partition.replace(partition); Ok(()) } + + fn height(&self) -> Option { + self.meta.height() + } + + fn has(&self, height: Height) -> bool { + self.meta.has(height) + } + fn needs(&self, height: Height) -> bool { + self.meta.needs(height) + } + + fn version(&self) -> Version { + self.meta.version() + } } impl Clone for Store @@ -276,13 +301,13 @@ where fn clone(&self) -> Self { Self { meta: self.meta.clone(), - name: self.name.clone(), + name: self.name, keyspace: self.keyspace.clone(), partition: None, rtx: self.keyspace.read_tx(), puts: self.puts.clone(), dels: self.dels.clone(), - bloom_filter_bits: self.bloom_filter_bits, + bloom_filters: self.bloom_filters, } } } diff --git a/crates/brk_store/src/meta.rs b/crates/brk_store/src/meta.rs index bf052f9a4..5597afc14 100644 --- a/crates/brk_store/src/meta.rs +++ b/crates/brk_store/src/meta.rs @@ -3,8 +3,8 @@ use std::{ path::{Path, PathBuf}, }; -use brk_core::{Result, Version, copy_first_8bytes}; -use fjall::{TransactionalKeyspace, TransactionalPartitionHandle}; +use brk_core::{Result, Version}; +use fjall::{PersistMode, TransactionalKeyspace, TransactionalPartitionHandle}; use super::Height; @@ -13,7 +13,6 @@ pub struct StoreMeta { pathbuf: PathBuf, version: Version, height: Option, - len: usize, } impl StoreMeta { @@ -40,17 +39,14 @@ impl StoreMeta { fs::remove_dir_all(path)?; fs::create_dir(path)?; keyspace.delete_partition(partition)?; - keyspace.persist(fjall::PersistMode::SyncAll)?; + keyspace.persist(PersistMode::SyncAll)?; partition = open_partition_handle()?; } - let len = Self::read_length_(path); - let slf = Self { pathbuf: path.to_owned(), version, height: Height::try_from(Self::path_height_(path).as_path()).ok(), - len, }; slf.version.write(&slf.path_version())?; @@ -58,28 +54,17 @@ impl StoreMeta { Ok((slf, partition)) } - pub fn len(&self) -> usize { - self.len + pub fn version(&self) -> Version { + self.version } - // pub fn is_empty(&self) -> bool { - // self.len() == 0 - // } - - // pub fn version(&self) -> Version { - // self.version - // } - - pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> { - self.len = len; - self.write_length()?; + pub fn export(&mut self, height: Height) -> io::Result<()> { self.height = Some(height); height.write(&self.path_height()) } pub fn reset(&mut self) { self.height.take(); - self.len = 0 } pub fn path(&self) -> &Path { @@ -108,19 +93,4 @@ impl StoreMeta { fn path_height_(path: &Path) -> PathBuf { path.join("height") } - - fn read_length_(path: &Path) -> usize { - fs::read(Self::path_length(path)) - .map(|v| usize::from_ne_bytes(copy_first_8bytes(v.as_slice()).unwrap())) - .unwrap_or_default() - } - fn write_length(&self) -> io::Result<()> { - Self::write_length_(&self.pathbuf, self.len) - } - fn write_length_(path: &Path, len: usize) -> io::Result<()> { - fs::write(Self::path_length(path), len.to_ne_bytes()) - } - fn path_length(path: &Path) -> PathBuf { - path.join("length") - } } diff --git a/crates/brk_store/src/trait.rs b/crates/brk_store/src/trait.rs new file mode 100644 index 000000000..5b6d06499 --- /dev/null +++ b/crates/brk_store/src/trait.rs @@ -0,0 +1,19 @@ +use brk_core::{Height, Result, Version}; + +pub trait AnyStore { + fn commit(&mut self, height: Height) -> Result<()>; + + fn persist(&self) -> Result<()>; + + fn reset(&mut self) -> Result<()>; + + fn name(&self) -> &'static str; + + fn height(&self) -> Option; + + fn has(&self, height: Height) -> bool; + + fn needs(&self, height: Height) -> bool; + + fn version(&self) -> Version; +} diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index 8291a91e1..677712d17 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -16,7 +16,7 @@ brk_exit = { workspace = true } clap = { workspace = true } clap_derive = { workspace = true } log = { workspace = true } -memmap2 = "0.9.5" +memmap2 = "0.9.7" rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index b82a6ba19..fc5500411 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -1,16 +1,18 @@ use std::{fs, path::Path}; -use brk_core::{DateIndex, Height, Version}; +use brk_core::{DateIndex, Height, Printable, Version}; use brk_vec::{AnyVec, CollectableVec, Format, GenericStoredVec, StoredVec, VecIterator}; +type I = DateIndex; #[allow(clippy::upper_case_acronyms)] -type VEC = StoredVec; +type VEC = StoredVec; fn main() -> Result<(), Box> { let _ = fs::remove_dir_all("./vec"); + let _ = fs::remove_file("./vec"); let version = Version::TWO; - let format = Format::Compressed; + let format = Format::Raw; { let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?; @@ -21,6 +23,8 @@ fn main() -> Result<(), Box> { let mut iter = vec.into_iter(); dbg!(iter.get(0.into())); + dbg!(iter.get(1.into())); + dbg!(iter.get(2.into())); dbg!(iter.get(20.into())); dbg!(iter.get(21.into())); @@ -38,6 +42,9 @@ fn main() -> Result<(), Box> { dbg!(iter.get(0.into())); dbg!(iter.get(1.into())); dbg!(iter.get(2.into())); + dbg!(iter.get(3.into())); + dbg!(iter.get(4.into())); + dbg!(iter.get(5.into())); dbg!(iter.get(20.into())); dbg!(iter.get(20.into())); dbg!(iter.get(0.into())); diff --git a/crates/brk_vec/src/structs/header.rs b/crates/brk_vec/src/structs/header.rs index 7a66793aa..30bd0c936 100644 --- a/crates/brk_vec/src/structs/header.rs +++ b/crates/brk_vec/src/structs/header.rs @@ -7,7 +7,6 @@ use std::{ use arc_swap::ArcSwap; use brk_core::{Error, Height, Result, Version}; -use memmap2::Mmap; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -31,8 +30,12 @@ impl Header { }) } - pub fn import_and_verify(mmap: &Mmap, vec_version: Version, format: Format) -> Result { - let inner = HeaderInner::import_and_verify(mmap, vec_version, format)?; + pub fn import_and_verify( + file: &mut File, + vec_version: Version, + format: Format, + ) -> Result { + let inner = HeaderInner::import_and_verify(file, vec_version, format)?; Ok(Self { inner: Arc::new(ArcSwap::from_pointee(inner)), modified: false, @@ -100,7 +103,6 @@ impl HeaderInner { compressed: ZeroCopyBool::from(format), }; header.write(file)?; - // dbg!(file.bytes().map(|b| b.unwrap()).collect::>()); file.seek(SeekFrom::End(0))?; Ok(header) } @@ -109,13 +111,22 @@ impl HeaderInner { file.write_all_at(self.as_bytes(), 0) } - pub fn import_and_verify(mmap: &Mmap, vec_version: Version, format: Format) -> Result { - if mmap.len() < HEADER_OFFSET { + pub fn import_and_verify( + file: &mut File, + vec_version: Version, + format: Format, + ) -> Result { + let len = file.metadata()?.len(); + + if len < HEADER_OFFSET as u64 { return Err(Error::WrongLength); } - // dbg!(mmap.len()); - let header = HeaderInner::read_from_bytes(&mmap[..HEADER_OFFSET])?; - // dbg!(&header); + + let mut buf = [0; HEADER_OFFSET]; + file.read_exact_at(&mut buf, 0)?; + + let header = HeaderInner::read_from_bytes(&buf)?; + if header.header_version != HEADER_VERSION { return Err(Error::DifferentVersion { found: header.header_version, @@ -136,6 +147,7 @@ impl HeaderInner { { return Err(Error::DifferentCompressionMode); } + Ok(header) } } diff --git a/crates/brk_vec/src/traits/any.rs b/crates/brk_vec/src/traits/any.rs index e9d35c1e2..84c516788 100644 --- a/crates/brk_vec/src/traits/any.rs +++ b/crates/brk_vec/src/traits/any.rs @@ -1,7 +1,16 @@ -use brk_core::Version; +use brk_core::{Height, Version}; use super::{BoxedVecIterator, StoredIndex, StoredType}; +pub fn i64_to_usize(i: i64, len: usize) -> usize { + if i >= 0 { + (i as usize).min(len) + } else { + let v = len as i64 + i; + if v < 0 { 0 } else { v as usize } + } +} + pub trait AnyVec: Send + Sync { fn version(&self) -> Version; fn name(&self) -> &str; @@ -11,10 +20,10 @@ pub trait AnyVec: Send + Sync { } fn index_type_to_string(&self) -> &'static str; fn value_type_to_size_of(&self) -> usize; - fn etag(&self, to: Option) -> String { + fn etag(&self, height: Height, to: Option) -> String { let len = self.len(); format!( - "{}-{:?}", + "{}-{}-{}", to.map_or(len, |to| { if to.is_negative() { len.checked_sub(to.unsigned_abs() as usize) @@ -23,9 +32,16 @@ pub trait AnyVec: Send + Sync { to as usize } }), - self.version() + u64::from(self.version()), + u32::from(height), ) } + + #[inline] + fn i64_to_usize(&self, i: i64) -> usize { + let len = self.len(); + i64_to_usize(i, len) + } } pub trait AnyIterableVec: AnyVec { diff --git a/crates/brk_vec/src/traits/collectable.rs b/crates/brk_vec/src/traits/collectable.rs index c643a9d73..87d7b840b 100644 --- a/crates/brk_vec/src/traits/collectable.rs +++ b/crates/brk_vec/src/traits/collectable.rs @@ -1,5 +1,7 @@ use brk_core::{Error, Result}; +use crate::i64_to_usize; + use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType}; pub trait CollectableVec: AnyVec + AnyIterableVec @@ -8,6 +10,10 @@ where I: StoredIndex, T: StoredType, { + fn collect(&self) -> Result> { + self.collect_range(None, None) + } + fn collect_range(&self, from: Option, to: Option) -> Result> { let len = self.len(); let from = from.unwrap_or_default(); @@ -20,12 +26,12 @@ where Ok(self .iter_at_(from) .take(to - from) - .map(|(_, v)| v.into_inner()) + .map(|(_, v)| v.into_owned()) .collect::>()) } #[inline] - fn i64_to_usize(i: i64, len: usize) -> usize { + fn i64_to_usize_(i: i64, len: usize) -> usize { if i >= 0 { (i as usize).min(len) } else { @@ -34,27 +40,19 @@ where } } - fn range_count(from: Option, to: Option, len: usize) -> usize { - let from = from.map(|i| Self::i64_to_usize(i, len)); - let to = to.map(|i| Self::i64_to_usize(i, len)); - (from.unwrap_or_default()..to.unwrap_or(len)).count() - } - - #[doc(hidden)] fn collect_signed_range(&self, from: Option, to: Option) -> Result> { - let len = self.len(); - let from = from.map(|i| Self::i64_to_usize(i, len)); - let to = to.map(|i| Self::i64_to_usize(i, len)); + let from = from.map(|i| self.i64_to_usize(i)); + let to = to.map(|i| self.i64_to_usize(i)); self.collect_range(from, to) } #[inline] fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { - self.collect_signed_range(from, to)? + self.collect_range(from, to)? .into_iter() .map(|v| serde_json::to_value(v).map_err(Error::from)) .collect::>>() @@ -72,7 +70,18 @@ where pub trait AnyCollectableVec: AnyVec { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result>; + + fn range_count(&self, from: Option, to: Option) -> usize { + let len = self.len(); + let from = from.map(|i| i64_to_usize(i, len)); + let to = to.map(|i| i64_to_usize(i, len)); + (from.unwrap_or_default()..to.unwrap_or(len)).count() + } + + fn range_weight(&self, from: Option, to: Option) -> usize { + self.range_count(from, to) * self.value_type_to_size_of() + } } diff --git a/crates/brk_vec/src/traits/generic.rs b/crates/brk_vec/src/traits/generic.rs index 38955e43a..4b0ffd2cf 100644 --- a/crates/brk_vec/src/traits/generic.rs +++ b/crates/brk_vec/src/traits/generic.rs @@ -1,12 +1,11 @@ use std::{ + borrow::Cow, fs::{File, OpenOptions}, io::{self, Seek, SeekFrom, Write}, path::{Path, PathBuf}, - sync::Arc, }; -use arc_swap::ArcSwap; -use brk_core::{Result, Value}; +use brk_core::Result; use memmap2::Mmap; use crate::{AnyVec, HEADER_OFFSET, Header}; @@ -21,6 +20,10 @@ where { const SIZE_OF_T: usize = size_of::(); + #[inline] + fn unwrap_read(&self, index: I, mmap: &Mmap) -> T { + self.read(index, mmap).unwrap().unwrap() + } #[inline] fn read(&self, index: I, mmap: &Mmap) -> Result> { self.read_(index.to_usize()?, mmap) @@ -28,12 +31,12 @@ where fn read_(&self, index: usize, mmap: &Mmap) -> Result>; #[inline] - fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { + fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { self.get_or_read_(index.to_usize()?, mmap) } #[inline] - fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result>> { - let stored_len = self.stored_len_(mmap); + fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result>> { + let stored_len = self.stored_len(); if index >= stored_len { let pushed = self.pushed(); @@ -41,9 +44,9 @@ where if j >= pushed.len() { return Ok(None); } - Ok(pushed.get(j).map(Value::Ref)) + Ok(pushed.get(j).map(Cow::Borrowed)) } else { - Ok(self.read_(index, mmap)?.map(Value::Owned)) + Ok(self.read_(index, mmap)?.map(Cow::Owned)) } } @@ -56,10 +59,7 @@ where format!("{}_to_{}", I::to_string(), self.name()) } - fn mmap(&self) -> &ArcSwap; - fn stored_len(&self) -> usize; - fn stored_len_(&self, mmap: &Mmap) -> usize; fn pushed(&self) -> &[T]; #[inline] @@ -111,7 +111,7 @@ where fn file_set_len(&mut self, file: &mut File, len: u64) -> Result<()> { Self::file_set_len_(file, len)?; - self.update_mmap(file) + Ok(()) } fn file_set_len_(file: &mut File, len: u64) -> Result<()> { file.set_len(len)?; @@ -121,13 +121,13 @@ where fn file_write_all(&mut self, file: &mut File, buf: &[u8]) -> Result<()> { file.write_all(buf)?; - self.update_mmap(file) + file.flush()?; + Ok(()) } fn file_truncate_and_write_all(&mut self, file: &mut File, len: u64, buf: &[u8]) -> Result<()> { Self::file_set_len_(file, len)?; - file.write_all(buf)?; - self.update_mmap(file) + self.file_write_all(file, buf) } fn reset(&mut self) -> Result<()>; @@ -138,14 +138,10 @@ where self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[]) } - fn new_mmap(file: &File) -> Result> { - Ok(Arc::new(unsafe { Mmap::map(file)? })) - } - - fn update_mmap(&mut self, file: &File) -> Result<()> { - let mmap = Self::new_mmap(file)?; - self.mmap().store(mmap); - Ok(()) + #[inline] + fn create_mmap(&self) -> Result { + let file = self.open_file()?; + unsafe { Mmap::map(&file).map_err(|e| e.into()) } } #[inline] diff --git a/crates/brk_vec/src/traits/iterator.rs b/crates/brk_vec/src/traits/iterator.rs index 3e0d8ef06..097e513a7 100644 --- a/crates/brk_vec/src/traits/iterator.rs +++ b/crates/brk_vec/src/traits/iterator.rs @@ -1,6 +1,6 @@ -use std::iter::Skip; +use std::{borrow::Cow, iter::Skip}; -use brk_core::{Printable, Value}; +use brk_core::Printable; use super::{StoredIndex, StoredType}; @@ -34,7 +34,7 @@ pub trait BaseVecIterator: Iterator { } } -pub trait VecIterator<'a>: BaseVecIterator)> { +pub trait VecIterator<'a>: BaseVecIterator)> { type I: StoredIndex; type T: StoredType + 'a; @@ -44,12 +44,12 @@ pub trait VecIterator<'a>: BaseVecIterator)> } #[inline] - fn get_(&mut self, i: usize) -> Option> { + fn get_(&mut self, i: usize) -> Option> { self.next_at(i).map(|(_, v)| v) } #[inline] - fn get(&mut self, i: Self::I) -> Option> { + fn get(&mut self, i: Self::I) -> Option> { self.get_(i.unwrap_to_usize()) } @@ -62,15 +62,15 @@ pub trait VecIterator<'a>: BaseVecIterator)> fn unwrap_get_inner_(&mut self, i: usize) -> Self::T { self.get_(i) .unwrap_or_else(|| { - dbg!(self.name(), i, self.len()); + dbg!(self.name(), i, self.len(), Self::I::to_string()); panic!("unwrap_get_inner_") }) - .into_inner() + .into_owned() } #[inline] fn get_inner(&mut self, i: Self::I) -> Option { - self.get_(i.unwrap_to_usize()).map(|v| v.into_inner()) + self.get_(i.unwrap_to_usize()).map(|v| v.into_owned()) } fn last(mut self) -> Option @@ -83,7 +83,7 @@ pub trait VecIterator<'a>: BaseVecIterator)> } let i = len - 1; self.set_(i); - self.next().map(|(i, v)| (i, Value::Owned(v.into_inner()))) + self.next() } fn index_type_to_string(&self) -> &'static str { @@ -93,7 +93,7 @@ pub trait VecIterator<'a>: BaseVecIterator)> impl<'a, I, T, Iter> VecIterator<'a> for Iter where - Iter: BaseVecIterator)>, + Iter: BaseVecIterator)>, I: StoredIndex, T: StoredType + 'a, { @@ -102,4 +102,4 @@ where } pub type BoxedVecIterator<'a, I, T> = - Box)> + 'a>; + Box)> + 'a>; diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index 769e33bbb..d81485a0b 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -1,11 +1,12 @@ use std::{ + borrow::Cow, fs, mem, path::{Path, PathBuf}, sync::Arc, }; use arc_swap::{ArcSwap, Guard}; -use brk_core::{Error, Result, Value, Version}; +use brk_core::{Error, Result, Version}; use memmap2::Mmap; use rayon::prelude::*; use zstd::DEFAULT_COMPRESSION_LEVEL; @@ -56,7 +57,7 @@ where } pub fn import(parent: &Path, name: &str, version: Version) -> Result { - let inner = RawVec::import(parent, name, version)?; + let mut inner = RawVec::import(parent, name, version)?; let pages_meta = { let path = inner @@ -65,12 +66,19 @@ where if inner.is_empty() { let _ = fs::remove_file(&path); } - Arc::new(ArcSwap::new(Arc::new(CompressedPagesMetadata::read( - &path, - )?))) + CompressedPagesMetadata::read(&path)? }; - Ok(Self { inner, pages_meta }) + inner.set_stored_len(if let Some(last) = pages_meta.last() { + (pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize + } else { + 0 + }); + + Ok(Self { + inner, + pages_meta: Arc::new(ArcSwap::new(Arc::new(pages_meta))), + }) } fn decode_page(&self, page_index: usize, mmap: &Mmap) -> Result> { @@ -119,24 +127,16 @@ where zstd::encode_all(bytes.as_slice(), DEFAULT_COMPRESSION_LEVEL).unwrap() } - #[inline(always)] + #[inline] fn index_to_page_index(index: usize) -> usize { index / Self::PER_PAGE } - #[inline(always)] + #[inline] fn page_index_to_index(page_index: usize) -> usize { page_index * Self::PER_PAGE } - fn stored_len__(pages_meta: &Guard>) -> usize { - if let Some(last) = pages_meta.last() { - (pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize - } else { - 0 - } - } - #[inline] pub fn iter(&self) -> CompressedVecIterator<'_, I, T> { self.into_iter() @@ -179,22 +179,13 @@ where self.inner.mut_header() } - #[inline] - fn mmap(&self) -> &ArcSwap { - self.inner.mmap() - } - fn parent(&self) -> &Path { self.inner.parent() } #[inline] fn stored_len(&self) -> usize { - Self::stored_len__(&self.pages_meta.load()) - } - #[inline] - fn stored_len_(&self, _: &Mmap) -> usize { - self.stored_len() + self.inner.stored_len() } #[inline] @@ -222,6 +213,8 @@ where let stored_len = self.stored_len(); + let mut file = file_opt.unwrap_or(self.open_file()?); + let mut pages_meta = (**self.pages_meta.load()).clone(); let mut starting_page_index = pages_meta.len(); @@ -235,16 +228,13 @@ where let last_page_index = pages_meta.len() - 1; - values = Self::decode_page_( - stored_len, - last_page_index, - &self.mmap().load(), - &pages_meta, - ) - .inspect_err(|_| { - dbg!(last_page_index, &pages_meta); - }) - .unwrap(); + let mmap = unsafe { Mmap::map(&file)? }; + + values = Self::decode_page_(stored_len, last_page_index, &mmap, &pages_meta) + .inspect_err(|_| { + dbg!(last_page_index, &pages_meta); + }) + .unwrap(); truncate_at.replace(pages_meta.pop().unwrap().start); starting_page_index = last_page_index; @@ -286,8 +276,6 @@ where pages_meta.write()?; - let mut file = file_opt.unwrap_or(self.open_file()?); - if let Some(truncate_at) = truncate_at { self.file_set_len(&mut file, truncate_at)?; } @@ -323,7 +311,11 @@ where let page_index = Self::index_to_page_index(index); - let values = self.decode_page(page_index, &self.mmap().load())?; + let mut file = self.open_file()?; + + let mmap = unsafe { Mmap::map(&file)? }; + + let values = self.decode_page(page_index, &mmap)?; let mut buf = vec![]; let mut page = pages_meta.truncate(page_index).unwrap(); @@ -347,8 +339,6 @@ where self.pages_meta.store(Arc::new(pages_meta)); - let mut file = self.open_file()?; - self.file_truncate_and_write_all(&mut file, len, &buf)?; Ok(()) @@ -398,7 +388,7 @@ impl Clone for CompressedVec { #[derive(Debug)] pub struct CompressedVecIterator<'a, I, T> { vec: &'a CompressedVec, - guard: Guard>, + mmap: Mmap, decoded_page: Option<(usize, Vec)>, // second_decoded_page?: Option<(usize, Vec)>, pages_meta: Guard>, @@ -441,10 +431,10 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { - let mmap = &self.guard; + let mmap = &self.mmap; let i = self.index; let stored_len = self.stored_len; @@ -456,7 +446,7 @@ where self.vec .pushed() .get(j) - .map(|v| (I::from(i), Value::Ref(v))) + .map(|v| (I::from(i), Cow::Borrowed(v))) } else { let page_index = i / Self::PER_PAGE; @@ -476,7 +466,7 @@ where .unwrap() .1 .get(i % Self::PER_PAGE) - .map(|v| (I::from(i), Value::Owned(v.clone()))) + .map(|v| (I::from(i), Cow::Owned(v.clone()))) }; self.index += 1; @@ -490,19 +480,20 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = CompressedVecIterator<'a, I, T>; fn into_iter(self) -> Self::IntoIter { let pages_meta = self.pages_meta.load(); - let stored_len = CompressedVec::::stored_len__(&pages_meta); + let stored_len = self.stored_len(); + CompressedVecIterator { vec: self, - guard: self.mmap().load(), + mmap: self.create_mmap().unwrap(), decoded_page: None, pages_meta, - stored_len, index: 0, + stored_len, } } } @@ -527,8 +518,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/computed.rs b/crates/brk_vec/src/variants/computed.rs index aefca73b8..53cc45ff2 100644 --- a/crates/brk_vec/src/variants/computed.rs +++ b/crates/brk_vec/src/variants/computed.rs @@ -1,10 +1,10 @@ -use std::{fs, path::Path}; +use std::{borrow::Cow, fs, path::Path}; use brk_exit::Exit; use clap_derive::ValueEnum; use serde::{Deserialize, Serialize}; -use brk_core::{Result, StoredPhantom, Value, Version}; +use brk_core::{Result, StoredPhantom, Version}; use crate::{ AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, @@ -36,7 +36,12 @@ impl Computation { } #[derive(Clone)] -pub enum Dependencies { +pub enum Dependencies +where + S1T: Clone, + S2T: Clone, + S3T: Clone, +{ From1(BoxedAnyIterableVec, ComputeFrom1), From2( (BoxedAnyIterableVec, BoxedAnyIterableVec), @@ -60,7 +65,12 @@ pub type ComputedVecFrom3 = ComputedVec; #[derive(Clone)] -pub enum ComputedVec { +pub enum ComputedVec +where + S1T: Clone, + S2T: Clone, + S3T: Clone, +{ Eager { vec: EagerVec, deps: Dependencies, @@ -82,7 +92,7 @@ where S3T: StoredType, { pub fn forced_import_or_init_from_1( - mode: Computation, + computation: Computation, path: &Path, name: &str, version: Version, @@ -90,7 +100,7 @@ where source: BoxedAnyIterableVec, compute: ComputeFrom1, ) -> Result { - Ok(match mode { + Ok(match computation { Computation::Eager => Self::Eager { vec: EagerVec::forced_import(path, name, version, format)?, deps: Dependencies::From1(source, compute), @@ -104,7 +114,7 @@ where #[allow(clippy::too_many_arguments)] pub fn forced_import_or_init_from_2( - mode: Computation, + computation: Computation, path: &Path, name: &str, version: Version, @@ -113,7 +123,7 @@ where source2: BoxedAnyIterableVec, compute: ComputeFrom2, ) -> Result { - Ok(match mode { + Ok(match computation { Computation::Eager => Self::Eager { vec: EagerVec::forced_import(path, name, version, format)?, deps: Dependencies::From2((source1, source2), compute), @@ -127,7 +137,7 @@ where #[allow(clippy::too_many_arguments)] pub fn forced_import_or_init_from_3( - mode: Computation, + computation: Computation, path: &Path, name: &str, version: Version, @@ -137,7 +147,7 @@ where source3: BoxedAnyIterableVec, compute: ComputeFrom3, ) -> Result { - Ok(match mode { + Ok(match computation { Computation::Eager => Self::Eager { vec: EagerVec::forced_import(path, name, version, format)?, deps: Dependencies::From3((source1, source2, source3), compute), @@ -251,7 +261,12 @@ where } } -pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> { +pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> +where + S1T: Clone, + S2T: Clone, + S3T: Clone, +{ Eager(StoredVecIterator<'a, I, T>), LazyFrom1(LazyVecFrom1Iterator<'a, I, T, S1I, S1T>), LazyFrom2(LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>), @@ -270,7 +285,7 @@ where S3I: StoredIndex, S3T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { match self { Self::Eager(i) => i.next(), @@ -335,7 +350,7 @@ where S3I: StoredIndex, S3T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; fn into_iter(self) -> Self::IntoIter { @@ -382,8 +397,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/eager.rs b/crates/brk_vec/src/variants/eager.rs index d684403d7..7bd5d3a47 100644 --- a/crates/brk_vec/src/variants/eager.rs +++ b/crates/brk_vec/src/variants/eager.rs @@ -1,5 +1,6 @@ use core::error; use std::{ + borrow::Cow, cmp::Ordering, f32, fmt::Debug, @@ -7,10 +8,9 @@ use std::{ path::{Path, PathBuf}, }; -use arc_swap::ArcSwap; use brk_core::{ Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Error, Result, Sats, StoredF32, - StoredUsize, Value, Version, + StoredUsize, Version, }; use brk_exit::Exit; use log::info; @@ -23,7 +23,7 @@ use crate::{ const ONE_KIB: usize = 1024; const ONE_MIB: usize = ONE_KIB * ONE_KIB; -const MAX_CACHE_SIZE: usize = 210 * ONE_MIB; +const MAX_CACHE_SIZE: usize = 256 * ONE_MIB; const DCA_AMOUNT: Dollars = Dollars::mint(100.0); #[derive(Debug, Clone)] @@ -103,14 +103,10 @@ where self.0.path() } - pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { + pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { self.0.get_or_read(index, mmap) } - pub fn mmap(&self) -> &ArcSwap { - self.0.mmap() - } - pub fn inner_version(&self) -> Version { self.0.version() } @@ -213,7 +209,7 @@ where let index = max_from.min(A::from(self.len())); other.iter_at(index).try_for_each(|(a, b)| { - let (i, v) = t((a, b.into_inner(), self)); + let (i, v) = t((a, b.into_owned(), self)); self.forced_push_at(i, v, exit) })?; @@ -238,7 +234,7 @@ where let mut adder_iter = adder.iter(); added.iter_at(index).try_for_each(|(i, v)| { - let v = v.into_inner() + adder_iter.unwrap_get_inner(i); + let v = v.into_owned() + adder_iter.unwrap_get_inner(i); self.forced_push_at(i, v, exit) })?; @@ -265,7 +261,7 @@ where subtracted.iter_at(index).try_for_each(|(i, v)| { let v = v - .into_inner() + .into_owned() .checked_sub(subtracter_iter.unwrap_get_inner(i)) .unwrap(); @@ -302,7 +298,7 @@ where T::from(source.iter().unwrap_get_inner_(0)) }); } - let max = prev.clone().unwrap().max(T::from(v.into_inner())); + let max = prev.clone().unwrap().max(T::from(v.into_owned())); prev.replace(max.clone()); self.forced_push_at(i, max, exit) @@ -332,7 +328,7 @@ where let mut multiplier_iter = multiplier.iter(); multiplied.iter_at(index).try_for_each(|(i, v)| { - let v = v.into_inner() * multiplier_iter.unwrap_get_inner(i); + let v = v.into_owned() * multiplier_iter.unwrap_get_inner(i); self.forced_push_at(i, v.into(), exit) })?; @@ -416,7 +412,7 @@ where let mut divider_iter = divider.iter(); divided.iter_at(index).try_for_each(|(i, divided)| { - let divided = divided.into_inner(); + let divided = divided.into_owned(); let divider = divider_iter.unwrap_get_inner(i); let v = if as_percentage { @@ -451,7 +447,7 @@ where let index = max_from.min(I::from(self.len())); let mut close_iter = close.iter(); ath.iter_at(index).try_for_each(|(i, ath)| { - let ath = ath.into_inner(); + let ath = ath.into_owned(); if ath == Dollars::ZERO { self.forced_push_at(i, T::from(StoredF32::default()), exit) } else { @@ -479,11 +475,11 @@ where )?; let index = max_from.min( - VecIterator::last(self.0.into_iter()).map_or_else(T::default, |(_, v)| v.into_inner()), + VecIterator::last(self.0.into_iter()).map_or_else(T::default, |(_, v)| v.into_owned()), ); let mut prev_i = None; other.iter_at(index).try_for_each(|(v, i)| -> Result<()> { - let i = i.into_inner(); + let i = i.into_owned(); if prev_i.is_some_and(|prev_i| prev_i == i) { return Ok(()); } @@ -647,7 +643,7 @@ where self_to_other.iter_at(index).try_for_each(|(i, other)| { self.forced_push_at( i, - T::from(other_to_self_iter.unwrap_get_inner(other.into_inner()) == i), + T::from(other_to_self_iter.unwrap_get_inner(other.into_owned()) == i), exit, ) })?; @@ -715,7 +711,7 @@ where .unwrap() .iter_at(index) .try_for_each(|(i, v)| { - let mut sum = v.into_inner(); + let mut sum = v.into_owned(); others_iter.iter_mut().for_each(|iter| { sum = sum.clone() + iter.unwrap_get_inner(i); }); @@ -750,7 +746,7 @@ where .unwrap() .iter_at(index) .try_for_each(|(i, v)| { - let min = v.into_inner(); + let min = v.into_owned(); let min = others_iter .iter_mut() .map(|iter| iter.unwrap_get_inner(i)) @@ -787,7 +783,7 @@ where .unwrap() .iter_at(index) .try_for_each(|(i, v)| { - let max = v.into_inner(); + let max = v.into_owned(); let max = others_iter .iter_mut() .map(|iter| iter.unwrap_get_inner(i)) @@ -836,7 +832,7 @@ where let min_prev_i = min_i.unwrap_or_default().unwrap_to_usize(); let mut other_iter = source.iter(); source.iter_at(index).try_for_each(|(i, value)| { - let value = value.into_inner(); + let value = value.into_owned(); if min_i.is_none() || min_i.is_some_and(|min_i| min_i <= i) { if prev.is_none() { @@ -924,7 +920,7 @@ where let index = max_from.min(I::from(self.len())); let mut source_iter = source.iter(); source.iter_at(index).try_for_each(|(i, current)| { - let current = current.into_inner(); + let current = current.into_owned(); let prev = i .checked_sub(I::from(len)) @@ -963,7 +959,7 @@ where .unwrap_or_default(), ); - let last_value = f32::from(b.into_inner()); + let last_value = f32::from(b.into_owned()); let percentage_change = ((last_value / previous_value) - 1.0) * 100.0; @@ -999,7 +995,7 @@ where percentage_returns .iter_at(index) .try_for_each(|(i, percentage)| { - let percentage = percentage.into_inner(); + let percentage = percentage.into_owned(); let cagr = (((f32::from(percentage) / 100.0 + 1.0).powf(1.0 / years as f32)) - 1.0) * 100.0; @@ -1054,7 +1050,7 @@ impl EagerVec { let index = max_from.min(DateIndex::from(self.len())); closes.iter_at(index).try_for_each(|(i, closes)| { - let price = *closes.into_inner(); + let price = *closes.into_owned(); let i_usize = i.unwrap_to_usize(); if prev.is_none() { if i_usize == 0 { @@ -1102,7 +1098,7 @@ impl EagerVec { let index = max_from.min(DateIndex::from(self.len())); closes.iter_at(index).try_for_each(|(i, closes)| { - let price = *closes.into_inner(); + let price = *closes.into_owned(); let i_usize = i.unwrap_to_usize(); if prev.is_none() { if i_usize == 0 { @@ -1144,7 +1140,7 @@ impl EagerVec { let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap(); stacks.iter_at(index).try_for_each(|(i, stack)| { - let stack = stack.into_inner(); + let stack = stack.into_owned(); let mut avg_price = Dollars::from(f64::NAN); if i > first_price_date { avg_price = DCA_AMOUNT @@ -1175,7 +1171,7 @@ impl EagerVec { let from_usize = from.unwrap_to_usize(); stacks.iter_at(index).try_for_each(|(i, stack)| { - let stack = stack.into_inner(); + let stack = stack.into_owned(); let mut avg_price = Dollars::from(f64::NAN); if i >= from { avg_price = @@ -1204,7 +1200,7 @@ where let index = max_from.min(I::from(self.len())); sats.iter_at(index).try_for_each(|(i, sats)| { - let (i, v) = (i, Bitcoin::from(sats.into_inner())); + let (i, v) = (i, Bitcoin::from(sats.into_owned())); self.forced_push_at(i, v, exit) })?; @@ -1231,7 +1227,7 @@ where let index = max_from.min(I::from(self.len())); bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| { let dollars = price_iter.unwrap_get_inner(i); - let (i, v) = (i, *dollars * bitcoin.into_inner()); + let (i, v) = (i, *dollars * bitcoin.into_owned()); self.forced_push_at(i, v, exit) })?; @@ -1262,7 +1258,7 @@ where // bitcoin.iter_at(index).try_for_each(|(i, bitcoin, ..)| { // let height = i_to_height_iter.unwrap_get_inner(i); // let dollars = price_iter.unwrap_get_inner(height); -// let (i, v) = (i, *dollars * bitcoin.into_inner()); +// let (i, v) = (i, *dollars * bitcoin.into_owned()); // self.forced_push_at(i, v, exit) // })?; @@ -1275,7 +1271,7 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = StoredVecIterator<'a, I, T>; fn into_iter(self) -> Self::IntoIter { @@ -1335,8 +1331,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/indexed.rs b/crates/brk_vec/src/variants/indexed.rs index 460712b6b..6ab8c836b 100644 --- a/crates/brk_vec/src/variants/indexed.rs +++ b/crates/brk_vec/src/variants/indexed.rs @@ -1,7 +1,6 @@ -use std::{cmp::Ordering, fmt::Debug, path::Path}; +use std::{borrow::Cow, cmp::Ordering, fmt::Debug, path::Path}; -use arc_swap::ArcSwap; -use brk_core::{Error, Height, Result, Value, Version}; +use brk_core::{Error, Height, Result, Version}; use crate::{ AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Format, @@ -30,7 +29,12 @@ where } #[inline] - pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { + pub fn unwrap_read(&self, index: I, mmap: &Mmap) -> T { + self.0.unwrap_read(index, mmap) + } + + #[inline] + pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { self.0.get_or_read(index, mmap) } @@ -73,8 +77,8 @@ where self.0.header() } - pub fn mmap(&self) -> &ArcSwap { - self.0.mmap() + pub fn create_mmap(&self) -> Result { + self.0.create_mmap() } #[inline] @@ -138,7 +142,7 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = StoredVecIterator<'a, I, T>; fn into_iter(self) -> Self::IntoIter { @@ -166,8 +170,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/lazy1.rs b/crates/brk_vec/src/variants/lazy1.rs index b55bef61c..2ceb0f693 100644 --- a/crates/brk_vec/src/variants/lazy1.rs +++ b/crates/brk_vec/src/variants/lazy1.rs @@ -1,6 +1,6 @@ -use std::marker::PhantomData; +use std::{borrow::Cow, marker::PhantomData}; -use brk_core::{Result, Value, Version}; +use brk_core::{Result, Version}; use crate::{ AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, @@ -8,10 +8,13 @@ use crate::{ }; pub type ComputeFrom1 = - for<'a> fn(I, &mut dyn BaseVecIterator)>) -> Option; + for<'a> fn(I, &mut dyn BaseVecIterator)>) -> Option; #[derive(Clone)] -pub struct LazyVecFrom1 { +pub struct LazyVecFrom1 +where + S1T: Clone, +{ name: String, version: Version, source: BoxedAnyIterableVec, @@ -32,8 +35,8 @@ where source: BoxedAnyIterableVec, compute: ComputeFrom1, ) -> Self { - if source.index_type_to_string() != I::to_string() { - panic!("Should have same index"); + if I::to_string() != S1I::to_string() { + unreachable!() } Self { @@ -50,7 +53,10 @@ where } } -pub struct LazyVecFrom1Iterator<'a, I, T, S1I, S1T> { +pub struct LazyVecFrom1Iterator<'a, I, T, S1I, S1T> +where + S1T: Clone, +{ lazy: &'a LazyVecFrom1, source: BoxedVecIterator<'a, S1I, S1T>, index: usize, @@ -63,14 +69,14 @@ where S1I: StoredIndex, S1T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { if self.index >= self.len() { return None; } let index = I::from(self.index); - let opt = (self.lazy.compute)(index, &mut *self.source).map(|v| (index, Value::Owned(v))); + let opt = (self.lazy.compute)(index, &mut *self.source).map(|v| (index, Cow::Owned(v))); if opt.is_some() { self.index += 1; } @@ -108,7 +114,7 @@ where S1I: StoredIndex, S1T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = LazyVecFrom1Iterator<'a, I, T, S1I, S1T>; fn into_iter(self) -> Self::IntoIter { @@ -173,8 +179,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/lazy2.rs b/crates/brk_vec/src/variants/lazy2.rs index 2a244d061..2d5bb249b 100644 --- a/crates/brk_vec/src/variants/lazy2.rs +++ b/crates/brk_vec/src/variants/lazy2.rs @@ -1,6 +1,6 @@ -use std::marker::PhantomData; +use std::{borrow::Cow, marker::PhantomData}; -use brk_core::{Result, Value, Version}; +use brk_core::{Result, Version}; use crate::{ AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, @@ -9,12 +9,16 @@ use crate::{ pub type ComputeFrom2 = for<'a> fn( I, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, ) -> Option; #[derive(Clone)] -pub struct LazyVecFrom2 { +pub struct LazyVecFrom2 +where + S1T: Clone, + S2T: Clone, +{ name: String, version: Version, source1: BoxedAnyIterableVec, @@ -66,7 +70,11 @@ where } } -pub struct LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> { +pub struct LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> +where + S1T: Clone, + S2T: Clone, +{ lazy: &'a LazyVecFrom2, source1: BoxedVecIterator<'a, S1I, S1T>, source2: BoxedVecIterator<'a, S2I, S2T>, @@ -82,12 +90,12 @@ where S2I: StoredIndex, S2T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { let index = I::from(self.index); let opt = (self.lazy.compute)(index, &mut *self.source1, &mut *self.source2) - .map(|v| (index, Value::Owned(v))); + .map(|v| (index, Cow::Owned(v))); if opt.is_some() { self.index += 1; } @@ -140,7 +148,7 @@ where S2I: StoredIndex, S2T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>; fn into_iter(self) -> Self::IntoIter { @@ -222,8 +230,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/lazy3.rs b/crates/brk_vec/src/variants/lazy3.rs index c9a351fa0..8bf3d8594 100644 --- a/crates/brk_vec/src/variants/lazy3.rs +++ b/crates/brk_vec/src/variants/lazy3.rs @@ -1,6 +1,6 @@ -use std::marker::PhantomData; +use std::{borrow::Cow, marker::PhantomData}; -use brk_core::{Result, Value, Version}; +use brk_core::{Result, Version}; use crate::{ AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, @@ -9,13 +9,18 @@ use crate::{ pub type ComputeFrom3 = for<'a> fn( I, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, ) -> Option; #[derive(Clone)] -pub struct LazyVecFrom3 { +pub struct LazyVecFrom3 +where + S1T: Clone, + S2T: Clone, + S3T: Clone, +{ name: String, version: Version, source1: BoxedAnyIterableVec, @@ -73,7 +78,12 @@ where } } -pub struct LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> { +pub struct LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> +where + S1T: Clone, + S2T: Clone, + S3T: Clone, +{ lazy: &'a LazyVecFrom3, source1: BoxedVecIterator<'a, S1I, S1T>, source2: BoxedVecIterator<'a, S2I, S2T>, @@ -93,7 +103,7 @@ where S3I: StoredIndex, S3T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { let index = I::from(self.index); @@ -103,7 +113,7 @@ where &mut *self.source2, &mut *self.source3, ) - .map(|v| (index, Value::Owned(v))); + .map(|v| (index, Cow::Owned(v))); if opt.is_some() { self.index += 1; } @@ -166,7 +176,7 @@ where S3I: StoredIndex, S3T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; fn into_iter(self) -> Self::IntoIter { @@ -262,8 +272,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index a9c98a7df..5847979a8 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -1,14 +1,17 @@ use std::{ + borrow::Cow, fs::{self, File}, io, marker::PhantomData, mem, path::{Path, PathBuf}, - sync::Arc, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, }; -use arc_swap::{ArcSwap, Guard}; -use brk_core::{Error, Result, Value, Version}; +use brk_core::{Error, Result, Version}; use memmap2::Mmap; use rayon::prelude::*; @@ -23,10 +26,10 @@ const VERSION: Version = Version::ONE; pub struct RawVec { header: Header, parent: PathBuf, - name: String, - // Consider Arc>> for dataraces when reorg ? - mmap: Arc>, + name: &'static str, pushed: Vec, + local_stored_len: Option, + shared_stored_len: Arc, phantom: PhantomData, } @@ -54,16 +57,18 @@ where pub fn import(parent: &Path, name: &str, version: Version) -> Result { let path = Self::path_(parent, name); - let (mmap, header) = match Self::open_file_(&path) { + let (header, file) = match Self::open_file_(&path) { Ok(mut file) => { if file.metadata()?.len() == 0 { - let header = Header::create_and_write(&mut file, version, Format::Raw)?; - let mmap = Self::new_mmap(&file)?; - (mmap, header) + ( + Header::create_and_write(&mut file, version, Format::Raw)?, + Some(file), + ) } else { - let mmap = Self::new_mmap(&file)?; - let header = Header::import_and_verify(&mmap, version, Format::Raw)?; - (mmap, header) + ( + Header::import_and_verify(&mut file, version, Format::Raw)?, + Some(file), + ) } } Err(e) => match e.kind() { @@ -71,25 +76,36 @@ where fs::create_dir_all(Self::folder_(parent, name))?; let mut file = Self::open_file_(&path)?; let header = Header::create_and_write(&mut file, version, Format::Raw)?; - let mmap = Self::new_mmap(&file)?; - (mmap, header) + (header, None) + } + _ => { + return Err(e.into()); } - _ => return Err(e.into()), }, }; - let mmap = Arc::new(ArcSwap::new(mmap)); + let stored_len = if let Some(file) = file { + (file.metadata()?.len() as usize - HEADER_OFFSET) / Self::SIZE_OF_T + } else { + 0 + }; Ok(Self { - mmap, header, - name: name.to_string(), + name: Box::leak(Box::new(name.to_string())), parent: parent.to_owned(), pushed: vec![], + local_stored_len: Some(stored_len), + shared_stored_len: Arc::new(AtomicUsize::new(stored_len)), phantom: PhantomData, }) } + pub fn set_stored_len(&mut self, len: usize) { + self.local_stored_len.replace(len); + self.shared_stored_len.store(len, Ordering::Relaxed); + } + #[inline] pub fn iter(&self) -> RawVecIterator<'_, I, T> { self.into_iter() @@ -140,18 +156,10 @@ where &mut self.header } - #[inline] - fn mmap(&self) -> &ArcSwap { - &self.mmap - } - #[inline] fn stored_len(&self) -> usize { - self.stored_len_(&self.mmap.load()) - } - #[inline] - fn stored_len_(&self, mmap: &Mmap) -> usize { - (mmap.len() - HEADER_OFFSET) / Self::SIZE_OF_T + self.local_stored_len + .unwrap_or_else(|| self.shared_stored_len.load(Ordering::Relaxed)) } #[inline] @@ -193,9 +201,14 @@ where }; let mut file = file_opt.unwrap_or(self.open_file()?); - self.file_write_all(&mut file, &bytes)?; + if let Some(local_stored_len) = self.local_stored_len.as_mut() { + *local_stored_len += pushed_len; + } + self.shared_stored_len + .fetch_add(pushed_len, Ordering::Relaxed); + Ok(()) } @@ -211,6 +224,8 @@ where return Ok(()); } + self.set_stored_len(index); + let len = index * Self::SIZE_OF_T + HEADER_OFFSET; let mut file = self.open_file()?; @@ -220,6 +235,7 @@ where } fn reset(&mut self) -> Result<()> { + self.set_stored_len(0); self.reset_() } } @@ -236,7 +252,7 @@ where #[inline] fn name(&self) -> &str { - &self.name + self.name } #[inline] @@ -260,10 +276,11 @@ impl Clone for RawVec { Self { header: self.header.clone(), parent: self.parent.clone(), - name: self.name.clone(), - mmap: self.mmap.clone(), + name: self.name, pushed: vec![], phantom: PhantomData, + local_stored_len: None, + shared_stored_len: self.shared_stored_len.clone(), } } } @@ -271,7 +288,7 @@ impl Clone for RawVec { #[derive(Debug)] pub struct RawVecIterator<'a, I, T> { vec: &'a RawVec, - guard: Guard>, + mmap: Mmap, index: usize, } @@ -301,15 +318,14 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { - let mmap = &self.guard; let index = self.index; let opt = self .vec - .get_or_read_(index, mmap) + .get_or_read_(index, &self.mmap) .unwrap() .map(|v| (I::from(index), v)); @@ -326,13 +342,13 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = RawVecIterator<'a, I, T>; fn into_iter(self) -> Self::IntoIter { RawVecIterator { vec: self, - guard: self.mmap.load(), + mmap: self.create_mmap().unwrap(), index: 0, } } @@ -358,8 +374,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/stored.rs b/crates/brk_vec/src/variants/stored.rs index 7283577ff..20f81283f 100644 --- a/crates/brk_vec/src/variants/stored.rs +++ b/crates/brk_vec/src/variants/stored.rs @@ -1,7 +1,9 @@ -use std::path::{Path, PathBuf}; +use std::{ + borrow::Cow, + path::{Path, PathBuf}, +}; -use arc_swap::ArcSwap; -use brk_core::{Result, Value, Version}; +use brk_core::{Result, Version}; use memmap2::Mmap; use crate::{ @@ -72,14 +74,6 @@ where } } - #[inline] - fn mmap(&self) -> &ArcSwap { - match self { - StoredVec::Raw(v) => v.mmap(), - StoredVec::Compressed(v) => v.mmap(), - } - } - #[inline] fn parent(&self) -> &Path { match self { @@ -95,13 +89,6 @@ where StoredVec::Compressed(v) => v.stored_len(), } } - #[inline] - fn stored_len_(&self, mmap: &Mmap) -> usize { - match self { - StoredVec::Raw(v) => v.stored_len_(mmap), - StoredVec::Compressed(v) => v.stored_len_(mmap), - } - } #[inline] fn pushed(&self) -> &[T] { @@ -195,7 +182,7 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); fn next(&mut self) -> Option { match self { Self::Compressed(i) => i.next(), @@ -238,7 +225,7 @@ where I: StoredIndex, T: StoredType, { - type Item = (I, Value<'a, T>); + type Item = (I, Cow<'a, T>); type IntoIter = StoredVecIterator<'a, I, T>; fn into_iter(self) -> Self::IntoIter { @@ -269,8 +256,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/websites/default/index.html b/websites/default/index.html index 1d343a71d..666c1b12a 100644 --- a/websites/default/index.html +++ b/websites/default/index.html @@ -36,8 +36,9 @@ line-height: 1.5; -webkit-text-size-adjust: 100%; tab-size: 4; - font-family: "Geist mono", ui-monospace, SFMono-Regular, Menlo, Monaco, - Consolas, "Liberation Mono", "Courier New", monospace; + font-family: + "Geist mono", ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, + "Liberation Mono", "Courier New", monospace; font-feature-settings: "ss03"; -webkit-tap-highlight-color: transparent; } @@ -80,9 +81,9 @@ kbd, samp, pre { - font-family: var(--default-mono-font-family), ui-monospace, - SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", - "Courier New", monospace; + font-family: + var(--default-mono-font-family), ui-monospace, SFMono-Regular, Menlo, + Monaco, Consolas, "Liberation Mono", "Courier New", monospace; font-feature-settings: var( --default-mono-font-feature-settings, normal @@ -1365,6 +1366,75 @@ } } } + + #explorer { + --cube: 4.5rem; + + #chain { + display: flex; + flex-direction: column-reverse; + gap: calc(var(--cube) * 0.66); + padding: 2rem; + + .cube { + width: var(--cube); + height: var(--cube); + overflow: hidden; + font-size: var(--font-size-sm); + line-height: var(--line-height-sm); + --face-color: var(--border-color); + color: var(--color); + + .face { + transform-origin: 0 0; + position: absolute; + width: var(--cube); + height: var(--cube); + padding: 0.1rem; + } + .right { + background-color: oklch( + from var(--face-color) calc(l - 0.05) c h + ); + transform: rotate(-30deg) skewX(-30deg) + translate(calc(var(--cube) * 1.3), calc(var(--cube) * 1.725)) + scaleY(0.864); + } + .top { + background-color: oklch( + from var(--face-color) calc(l + 0.05) c h + ); + transform: rotate(30deg) skew(-30deg) + translate(calc(var(--cube) * 0.99), calc(var(--cube) * -0.265)) + scaleY(0.864); + justify-content: center; + align-items: center; + text-align: center; + display: flex; + font-weight: 900; + text-transform: uppercase; + font-size: var(--font-size-xs); + line-height: var(--line-height-xs); + } + .left { + font-size: var(--font-size-xs); + line-height: var(--line-height-xs); + background-color: var(--face-color); + /* background-color: var(--amber); */ + transform: rotate(30deg) skewX(30deg) + translate(calc(var(--cube) * 0.3), calc(var(--cube) * 0.6)) + scaleY(0.864); + } + .fees { + display: flex; + flex-direction: column; + height: 100%; + justify-content: center; + align-items: center; + } + } + } + } @@ -1678,6 +1748,7 @@