From 700352ec45397caab431d51ebc079cd7eebdcaf2 Mon Sep 17 00:00:00 2001 From: nym21 Date: Wed, 30 Apr 2025 18:29:18 +0200 Subject: [PATCH] vec: caching only in iter --- Cargo.lock | 178 +++++++++--------- Cargo.toml | 4 +- crates/brk_cli/Cargo.toml | 2 +- .../brk_computer/src/storage/vecs/blocks.rs | 4 +- .../src/storage/vecs/grouped/builder.rs | 24 ++- .../src/storage/vecs/marketprice.rs | 5 +- .../src/storage/vecs/transactions.rs | 14 +- .../src/storage/vecs/vec/eager.rs | 30 +-- crates/brk_indexer/examples/main.rs | 2 +- crates/brk_indexer/src/indexes.rs | 79 ++++---- crates/brk_indexer/src/lib.rs | 69 ++++++- crates/brk_indexer/src/stores/mod.rs | 77 +++++--- crates/brk_indexer/src/vecs/base.rs | 12 +- crates/brk_indexer/src/vecs/mod.rs | 66 ++----- crates/brk_server/Cargo.toml | 2 +- crates/brk_vec/examples/main.rs | 53 +++--- crates/brk_vec/src/lib.rs | 45 +---- crates/brk_vec/src/traits/dynamic.rs | 67 +++---- crates/brk_vec/src/traits/generic.rs | 8 - crates/brk_vec/src/variants/compressed.rs | 177 +++++------------ crates/brk_vec/src/variants/raw.rs | 64 ++----- update.sh | 2 +- 22 files changed, 411 insertions(+), 573 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8723f6934..183d6f23e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ "rlimit", "serde", "serde_bytes", - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -494,7 +494,7 @@ dependencies = [ "fjall", "log", "rayon", - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -519,7 +519,7 @@ dependencies = [ "rayon", "serde", "serde_json", - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -574,7 +574,7 @@ dependencies = [ "rayon", "serde", "serde_json", - "zerocopy 0.8.24", + "zerocopy 0.8.25", "zstd", ] @@ -685,9 +685,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -737,7 +737,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -957,7 +957,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -968,7 +968,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1009,7 +1009,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1055,7 +1055,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1121,9 +1121,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "fjall" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958511f67d1f80e6bff9ffac05c626bb340d4602ca6ea5617d9901c218c894f0" +checksum = "b929b3db7be7d7b4d4df67fb016fc446b8f57507b48a82e69d2a30610e460f28" dependencies = [ "byteorder", "byteview", @@ -1513,7 +1513,7 @@ checksum = "199b7932d97e325aff3a7030e141eafe7f2c6268e1d1b24859b753a627f45254" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1599,9 +1599,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lsm-tree" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d58bdef2dcbf50fce9f343265bdbd7fb08a458d241eb837ce426be22d674b4" +checksum = "d0d03b764a7e3009cc4d314bfce42ce28b4a2c458fc7149b57817cbed7898f43" dependencies = [ "byteorder", "crossbeam-skiplist", @@ -1805,9 +1805,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "oxc" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409ba57c4752b7fab9609af78790aadeb541316933d1525484dad877988622f6" +checksum = "6f5cf75f3e465c16c27fe188143b2263b74b0729b3cf2044cd946452ce87ee93" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1843,14 +1843,14 @@ checksum = "4c0c893f53900e3fe01eca3d6d3b54085573c3e48fe25af9d57dd94ef600dcd3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "oxc_allocator" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43fe5f8d34c351e3c90b512046dbfff2ff87c8a1927ebe6945f1cdefcc3219d0" +checksum = "8d5e92c9020da9b683bcf06bec4895e618e5d86a290c6e8a805faa7fde701116" dependencies = [ "allocator-api2", "bumpalo", @@ -1862,9 +1862,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad2e9c3fb4b409b14fd791fbbf6ced9a81363364beedd48059bbb8f42bf4987" +checksum = "59d46e3425ff92b7edee4c9d299aff471b51fd813780d006b645243e7f522c3f" dependencies = [ "bitflags", "cow-utils", @@ -1879,20 +1879,20 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578b69ed39d814cb8396674e17a0f94f1c02bc25d226b265c2913bd6e84afffb" +checksum = "b1be1f451540565035dec927705c83826ca15e96d87d0352395c5e10d4822369" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "oxc_ast_visit" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f437ca959bf931d5facbc6df77c7491a4aec32df5b9f647ffa19c52055a7dc56" +checksum = "7db4c6ac94f2c157a9dd96d5715a656f049653c2088f8c241aea55bd9789b811" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1902,9 +1902,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecf0bce4034ebf17e7edb6f214fa0e6f8d134d1a3fb98408d6bcbef024a3a1" +checksum = "c5bf065a22ae9438554b88b664974be1bcb1060e997bb4ea6be27831af48badb" dependencies = [ "bitflags", "itertools", @@ -1917,9 +1917,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3361ed40dd6b4b6a7a4b44c175aa6beea68eefed68bc32a28684a0d1a00bc39" +checksum = "165f1993b33d4c07ec8e90c5522c6afe8985a463eca8545ac1c147863a86d472" dependencies = [ "bitflags", "cow-utils", @@ -1938,15 +1938,15 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d2d64d2e627796f73e37ccb51f1f87cd9de612245ef3f790051855587d3734e" +checksum = "5dc6d72677e31673f912d567f7d46d6390844a6ceabbc3bd4ef4a99b04178beb" [[package]] name = "oxc_diagnostics" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ece1486240cfeec516eaaa7332c96decb3a756d72e504bb63393507996c39d" +checksum = "5f53beee5473b6929be652ddfed20d3db3e6f96aea57f76a5c7f53ee1e7b458d" dependencies = [ "cow-utils", "oxc-miette", @@ -1954,9 +1954,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7d5ddf1cd783ed1a930ac6eedd4f73c6a5359beaabad7ee8333e366816291c" +checksum = "e6674bfaf4ffaa4723cf665e994b630a13cfdc641d2d3f208a010a9ec2bed27c" dependencies = [ "cow-utils", "num-bigint", @@ -1968,9 +1968,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62de94b10df7ab86a379f01b75076f6a762fc91a001678108318203fc0805ad9" +checksum = "ce1d1d67d2e951ee52e2c9d6e2995be7d54523574396d9c87423092ccb0f6b3c" [[package]] name = "oxc_index" @@ -1980,9 +1980,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d691c6609bca852ae1e0571de564de3b4d110a46c9877aa029e9dba56b76d1c" +checksum = "6da1b7049a4a0439401b6ae72f1375a0e66a38e4461fb235912cfe67331ce596" dependencies = [ "fixedbitset", "itertools", @@ -1997,9 +1997,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "294aa5790cc0beae7018d99dc5b389d373725ef7ea7ecef5d0ed18794b3414b3" +checksum = "c5bae5ecc9c768d15821994a8445e0e6aff763a9bb19a5635d34cab40423366f" dependencies = [ "cow-utils", "oxc_allocator", @@ -2019,9 +2019,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7defa49cc20f9e3c346674b06752cacaacce053cde69a04f0f85f31e00820f" +checksum = "67fd747c632d17d36fe496a514c4c87cb2b150f7358a1adb2c01d9d9018e8aa3" dependencies = [ "bitflags", "cow-utils", @@ -2042,9 +2042,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d37e8585630a167c4a2b121dcdab52bf13b71a6bd64260efe797956bd4f6d0" +checksum = "298174d22ce142abd379f6a0037d1f46b45e8b2beda894b636c18d7f7dc7645e" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -2058,9 +2058,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895130e4f197b77d9b54deffccc2ce8a9d977435c022a49ddecb1f467b946854" +checksum = "f446d2505831faebbe32679dfd3ea5ff2ebafce34ec1cace54541afd7e8102b4" dependencies = [ "itertools", "oxc_allocator", @@ -2094,9 +2094,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8350c982e6284f41668ebd3c714de22280a19f617c3cb6117bbc4e16afc6b13a" +checksum = "75b2d153db7fa3f82316d64f23da65588eabc7763e961c3fe9b4d6224fe5ae36" dependencies = [ "compact_str", "oxc-miette", @@ -2107,9 +2107,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e85d9197e08e176d6f58a3885d125738c46a69145410e82b3a32f676ddd5546" +checksum = "faf67148b2cef2b0f8df792e3bbda51957e4c39182e800b0370c60a98b682d1a" dependencies = [ "bitflags", "cow-utils", @@ -2128,11 +2128,10 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac43e668dcd3a0b17cf4f889feac5321e56430a0550205df2e85b3e9314f6caa" +checksum = "91b56ff860c0a85863cde254924ec2c67da24dbb7b5dda999cab4ae18cbfd245" dependencies = [ - "compact_str", "itoa", "oxc_allocator", "oxc_ast", @@ -2255,7 +2254,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2312,7 +2311,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -2334,7 +2333,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2628,7 +2627,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2701,7 +2700,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2816,9 +2815,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.100" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -2852,7 +2851,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2905,7 +2904,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2975,7 +2974,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2993,9 +2992,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.20" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" dependencies = [ "serde", "serde_spanned", @@ -3005,26 +3004,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" + [[package]] name = "tower" version = "0.5.2" @@ -3093,7 +3099,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3247,7 +3253,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -3269,7 +3275,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3310,7 +3316,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3321,7 +3327,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3474,11 +3480,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ - "zerocopy-derive 0.8.24", + "zerocopy-derive 0.8.25", ] [[package]] @@ -3489,18 +3495,18 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3520,7 +3526,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1b5f8c29a..cf631f6d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ byteview = "0.6.1" clap = { version = "4.5.37", features = ["derive", "string"] } color-eyre = "0.6.3" derive_deref = "1.1.1" -fjall = "2.9.0" +fjall = "2.10.0" jiff = "0.2.10" log = { version = "0.4.27" } minreq = { version = "2.13.4", features = ["https", "serde_json"] } @@ -42,7 +42,7 @@ rayon = "1.10.0" serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.140", features = ["float_roundtrip"] } tabled = "0.19.0" -zerocopy = { version = "0.8.24", features = ["derive"] } +zerocopy = { version = "0.8.25", features = ["derive"] } [workspace.metadata.release] shared-version = true diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 4d56d40fd..47c889df4 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -21,7 +21,7 @@ color-eyre = { workspace = true } log = { workspace = true } serde = { workspace = true } tabled = { workspace = true } -toml = "0.8.20" +toml = "0.8.22" [[bin]] name = "brk" diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 6059b2422..27eae7370 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -209,14 +209,14 @@ impl Vecs { self.difficultyepoch_to_timestamp.compute_transform( starting_indexes.difficultyepoch, indexes.difficultyepoch_to_first_height.vec(), - |(i, h, ..)| (i, height_to_timestamp_iter.get(h).unwrap().1.into_inner()), + |(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)), exit, )?; self.halvingepoch_to_timestamp.compute_transform( starting_indexes.halvingepoch, indexes.halvingepoch_to_first_height.vec(), - |(i, h, ..)| (i, height_to_timestamp_iter.get(h).unwrap().1.into_inner()), + |(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)), exit, )?; diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index 7abefed61..7744d757e 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -186,12 +186,12 @@ where .try_for_each(|(i, first_index)| -> Result<()> { let first_index = first_index.into_inner(); - let count_index = count_indexes_iter.get(i).unwrap().1.into_inner(); + let count_index = count_indexes_iter.unwrap_get_inner(i); if let Some(first) = self.first.as_mut() { let f = source_iter - .get(first_index) - .map_or(T::from(0_usize), |f| f.1.into_inner()); + .get_inner(first_index) + .unwrap_or_else(|| T::from(0_usize)); first.forced_push_at(index, f, exit)?; } @@ -201,15 +201,13 @@ where panic!("should compute last if count can be 0") } let last_index = first_index + (count_index - 1); - let v = source_iter - .get(last_index) - .context("to work") - .inspect_err(|_| { - dbg!(first_index, count_index, last_index); - }) - .unwrap() - .1 - .into_inner(); + let v = source_iter.unwrap_get_inner(last_index); + // .context("to work") + // .inspect_err(|_| { + // dbg!(first_index, count_index, last_index); + // }) + // .unwrap() + // .into_inner(); last.forced_push_at(index, v, exit)?; } @@ -357,7 +355,7 @@ where .try_for_each(|(i, first_index, ..)| -> Result<()> { let first_index = first_index.into_inner(); - let count_index = count_indexes_iter.get(i).unwrap().1.into_inner(); + let count_index = count_indexes_iter.unwrap_get_inner(i); if let Some(first) = self.first.as_mut() { let v = source_first_iter diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index 4d38057a8..931e598ec 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -342,9 +342,8 @@ impl Vecs { .get_height( h, t, - h.decremented().map(|prev_h| { - height_to_timestamp_iter.get(prev_h).unwrap().1.into_inner() - }), + h.decremented() + .map(|prev_h| height_to_timestamp_iter.unwrap_get_inner(prev_h)), ) .unwrap(); (h, ohlc) diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index d1e13f3c3..62799a44b 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -471,11 +471,7 @@ impl Vecs { indexer.vecs().height_to_first_txindex.vec(), indexer.vecs().txindex_to_txid.vec(), |txindex| { - let v = txindex_to_txversion_iter - .get(txindex) - .unwrap() - .1 - .into_inner(); + let v = txindex_to_txversion_iter.unwrap_get_inner(txindex); v == txversion }, exit, @@ -499,11 +495,7 @@ impl Vecs { starting_indexes.txindex, indexer.vecs().txindex_to_base_size.vec(), |(txindex, base_size, ..)| { - let total_size = txindex_to_total_size_iter - .get(txindex) - .unwrap() - .1 - .into_inner(); + let total_size = txindex_to_total_size_iter.unwrap_get_inner(txindex); // This is the exact definition of a weight unit, as defined by BIP-141 (quote above). let wu = usize::from(base_size) * 3 + usize::from(total_size); @@ -533,7 +525,7 @@ impl Vecs { |(inputindex, outputindex, ..)| { let value = if outputindex == OutputIndex::COINBASE { Sats::ZERO - } else if let Some((_, value)) = outputindex_to_value_iter.get(outputindex) { + } else if let Some(value) = outputindex_to_value_iter.get(outputindex) { value.into_inner() } else { dbg!(inputindex, outputindex, inputs_len); diff --git a/crates/brk_computer/src/storage/vecs/vec/eager.rs b/crates/brk_computer/src/storage/vecs/vec/eager.rs index 91e0a2013..8298bc91e 100644 --- a/crates/brk_computer/src/storage/vecs/vec/eager.rs +++ b/crates/brk_computer/src/storage/vecs/vec/eager.rs @@ -12,7 +12,6 @@ use brk_vec::{ Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, }; -use color_eyre::eyre::ContextCompat; use log::info; const ONE_KIB: usize = 1024; @@ -223,11 +222,7 @@ where if prev_i.is_some_and(|prev_i| prev_i == i) { return Ok(()); } - if self - .inner - .get(i)? - .is_none_or(|old_v| old_v.into_inner() > v) - { + if self.iter().get_inner(i).is_none_or(|old_v| old_v > v) { self.forced_push_at(i, v, exit)?; } prev_i.replace(i); @@ -350,8 +345,8 @@ where .iter_at(index) .try_for_each(|(i, first_index)| { let end = other_iter - .get(i + 1) - .map(|(_, v)| v.into_inner().unwrap_to_usize()) + .get_inner(i + 1) + .map(|v| v.unwrap_to_usize()) .unwrap_or_else(|| other_to_else.len()); let range = first_index.unwrap_to_usize()..end; @@ -387,14 +382,7 @@ where self_to_other.iter_at(index).try_for_each(|(i, other)| { self.forced_push_at( i, - T::from( - other_to_self_iter - .get(other.into_inner()) - .unwrap() - .1 - .into_inner() - == i, - ), + T::from(other_to_self_iter.unwrap_get_inner(other.into_inner()) == i), exit, ) })?; @@ -424,12 +412,12 @@ where first_indexes .iter_at(index) .try_for_each(|(i, first_index)| { - let count = *indexes_count_iter.get(i).unwrap().1.into_inner(); + let count = *indexes_count_iter.unwrap_get_inner(i); let first_index = first_index.unwrap_to_usize(); let range = first_index..first_index + count; let mut sum = T::from(0_usize); range.into_iter().for_each(|i| { - sum = sum.clone() + source_iter.get(T2::from(i)).unwrap().1.into_inner(); + sum = sum.clone() + source_iter.unwrap_get_inner(T2::from(i)); }); self.forced_push_at(i, sum, exit) })?; @@ -477,7 +465,7 @@ impl EagerVec { let mut price_iter = price.iter(); let index = max_from.min(Height::from(self.len())); bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| { - let dollars = price_iter.get(i).unwrap().1.into_inner(); + let dollars = price_iter.unwrap_get_inner(i); let (i, v) = (i, *dollars * bitcoin.into_inner()); self.forced_push_at(i, v, exit) })?; @@ -503,8 +491,8 @@ impl EagerVec { let mut price_iter = price.iter(); let index = max_from.min(TxIndex::from(self.len())); bitcoin.iter_at(index).try_for_each(|(i, bitcoin, ..)| { - let height = i_to_height_iter.get(i).unwrap().1.into_inner(); - let dollars = price_iter.get(height).unwrap().1.into_inner(); + let height = i_to_height_iter.unwrap_get_inner(i); + let dollars = price_iter.unwrap_get_inner(height); let (i, v) = (i, *dollars * bitcoin.into_inner()); self.forced_push_at(i, v, exit) })?; diff --git a/crates/brk_indexer/examples/main.rs b/crates/brk_indexer/examples/main.rs index 616932dcd..5a5ce1285 100644 --- a/crates/brk_indexer/examples/main.rs +++ b/crates/brk_indexer/examples/main.rs @@ -24,7 +24,7 @@ fn main() -> color_eyre::Result<()> { let outputs = Path::new("../../_outputs"); - let mut indexer = Indexer::new(outputs, false, true)?; + let mut indexer = Indexer::new(outputs, false, false)?; indexer.import_stores()?; indexer.import_vecs()?; diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index 5db6370ea..e55acab46 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -5,7 +5,7 @@ use brk_core::{ P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex, }; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; -use brk_vec::{Result, StoredIndex, StoredType, Value}; +use brk_vec::{StoredIndex, StoredType}; use color_eyre::eyre::ContextCompat; use crate::{IndexedVec, Stores, Vecs}; @@ -107,115 +107,116 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { }) .unwrap(); - vecs.height_to_blockhash.get(*height).map_or(true, |opt| { - opt.is_none_or(|saved_blockhash| { + vecs.height_to_blockhash + .iter() + .get(*height) + .is_none_or(|saved_blockhash| { let b = &rpc_blockhash != saved_blockhash.as_ref(); if b { dbg!(rpc_blockhash, saved_blockhash.as_ref()); } b }) - }) }) .unwrap_or(starting_height); Ok(Self { - emptyoutputindex: *starting_index( + emptyoutputindex: starting_index( &vecs.height_to_first_emptyoutputindex, &vecs.emptyoutputindex_to_txindex, height, - )? + ) .context("")?, height, - p2msindex: *starting_index( + p2msindex: starting_index( &vecs.height_to_first_p2msindex, &vecs.p2msindex_to_txindex, height, - )? + ) .context("")?, - opreturnindex: *starting_index( + opreturnindex: starting_index( &vecs.height_to_first_opreturnindex, &vecs.opreturnindex_to_txindex, height, - )? + ) .context("")?, - p2pk33index: *starting_index( + p2pk33index: starting_index( &vecs.height_to_first_p2pk33index, &vecs.p2pk33index_to_p2pk33bytes, height, - )? + ) .context("")?, - p2pk65index: *starting_index( + p2pk65index: starting_index( &vecs.height_to_first_p2pk65index, &vecs.p2pk65index_to_p2pk65bytes, height, - )? + ) .context("")?, - p2pkhindex: *starting_index( + p2pkhindex: starting_index( &vecs.height_to_first_p2pkhindex, &vecs.p2pkhindex_to_p2pkhbytes, height, - )? + ) .context("")?, - p2shindex: *starting_index( + p2shindex: starting_index( &vecs.height_to_first_p2shindex, &vecs.p2shindex_to_p2shbytes, height, - )? + ) .context("")?, - p2trindex: *starting_index( + p2trindex: starting_index( &vecs.height_to_first_p2trindex, &vecs.p2trindex_to_p2trbytes, height, - )? + ) .context("")?, - p2wpkhindex: *starting_index( + p2wpkhindex: starting_index( &vecs.height_to_first_p2wpkhindex, &vecs.p2wpkhindex_to_p2wpkhbytes, height, - )? + ) .context("")?, - p2wshindex: *starting_index( + p2wshindex: starting_index( &vecs.height_to_first_p2wshindex, &vecs.p2wshindex_to_p2wshbytes, height, - )? + ) .context("")?, - p2aindex: *starting_index( + p2aindex: starting_index( &vecs.height_to_first_p2aindex, &vecs.p2aindex_to_p2abytes, height, - )? + ) .context("")?, - txindex: *starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)? + txindex: starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height) .context("")?, - inputindex: *starting_index( + inputindex: starting_index( &vecs.height_to_first_inputindex, &vecs.inputindex_to_outputindex, height, - )? + ) .context("")?, - outputindex: *starting_index( + outputindex: starting_index( &vecs.height_to_first_outputindex, &vecs.outputindex_to_value, height, - )? + ) .context("")?, - unknownoutputindex: *starting_index( + unknownoutputindex: starting_index( &vecs.height_to_first_unknownoutputindex, &vecs.unknownoutputindex_to_txindex, height, - )? + ) .context("")?, }) } } -pub fn starting_index<'a, I, T>( - height_to_index: &'a IndexedVec, - index_to_else: &'a IndexedVec, +pub fn starting_index( + height_to_index: &IndexedVec, + index_to_else: &IndexedVec, starting_height: Height, -) -> Result>> +) -> Option where I: StoredType + StoredIndex + From, T: StoredType, @@ -224,8 +225,8 @@ where .height() .is_ok_and(|h| h + 1_u32 == starting_height) { - Ok(Some(Value::Owned(I::from(index_to_else.len())))) + Some(I::from(index_to_else.len())) } else { - height_to_index.get(starting_height) + height_to_index.iter().get_inner(starting_height) } } diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 596d0c516..b0708e07c 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -18,7 +18,7 @@ pub use brk_parser::*; use bitcoin::{Transaction, TxIn, TxOut}; use brk_exit::Exit; -use brk_vec::Compressed; +use brk_vec::{Compressed, DynamicVec}; use color_eyre::eyre::{ContextCompat, eyre}; use fjall::TransactionalKeyspace; use log::{error, info}; @@ -242,7 +242,11 @@ impl Indexer { }) }); + let input_source_vec_handle = scope.spawn(|| { + let txindex_to_first_outputindex_mmap = vecs + .txindex_to_first_outputindex.vec().mmap().load(); + inputs .into_par_iter() .enumerate() @@ -273,13 +277,11 @@ impl Indexer { let vout = Vout::from(outpoint.vout); - let outputindex = *vecs - .txindex_to_first_outputindex - .get(prev_txindex)? + let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, &txindex_to_first_outputindex_mmap)? .context("Expect outputindex to not be none") .inspect_err(|_| { dbg!(outpoint.txid, prev_txindex, vout); - })? + })?.into_inner() + vout; Ok((inputindex, InputSource::PreviousBlock(( @@ -305,6 +307,16 @@ impl Indexer { }); let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| { + let p2pk65index_to_p2pk65bytes_mmap = vecs + .p2pk65index_to_p2pk65bytes.vec().mmap().load(); + let p2pk33index_to_p2pk33bytes_mmap = vecs.p2pk33index_to_p2pk33bytes.vec().mmap().load(); + let p2pkhindex_to_p2pkhbytes_mmap = vecs.p2pkhindex_to_p2pkhbytes.vec().mmap().load(); + let p2shindex_to_p2shbytes_mmap = vecs.p2shindex_to_p2shbytes.vec().mmap().load(); + let p2wpkhindex_to_p2wpkhbytes_mmap = vecs.p2wpkhindex_to_p2wpkhbytes.vec().mmap().load(); + let p2wshindex_to_p2wshbytes_mmap = vecs.p2wshindex_to_p2wshbytes.vec().mmap().load(); + let p2trindex_to_p2trbytes_mmap = vecs.p2trindex_to_p2trbytes.vec().mmap().load(); + let p2aindex_to_p2abytes_mmap = vecs.p2aindex_to_p2abytes.vec().mmap().load(); + outputs .into_par_iter() .enumerate() @@ -349,8 +361,43 @@ impl Indexer { if let Some(Some(outputtypeindex)) = check_collisions.then_some(outputtypeindex_opt) { let addressbytes = address_bytes_res.as_ref().unwrap(); - let prev_addressbytes_opt = - vecs.get_addressbytes(outputtype, outputtypeindex)?; + let prev_addressbytes_opt = match outputtype { + OutputType::P2PK65 => vecs + .p2pk65index_to_p2pk65bytes + .get_or_read(outputtypeindex.into(), &p2pk65index_to_p2pk65bytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::P2PK33 => vecs + .p2pk33index_to_p2pk33bytes + .get_or_read(outputtypeindex.into(), &p2pk33index_to_p2pk33bytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::P2PKH => vecs + .p2pkhindex_to_p2pkhbytes + .get_or_read(outputtypeindex.into(), &p2pkhindex_to_p2pkhbytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::P2SH => vecs + .p2shindex_to_p2shbytes + .get_or_read(outputtypeindex.into(), &p2shindex_to_p2shbytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::P2WPKH => vecs + .p2wpkhindex_to_p2wpkhbytes + .get_or_read(outputtypeindex.into(), &p2wpkhindex_to_p2wpkhbytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::P2WSH => vecs + .p2wshindex_to_p2wshbytes + .get_or_read(outputtypeindex.into(), &p2wshindex_to_p2wshbytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::P2TR => vecs + .p2trindex_to_p2trbytes + .get_or_read(outputtypeindex.into(), &p2trindex_to_p2trbytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::P2A => vecs + .p2aindex_to_p2abytes + .get_or_read(outputtypeindex.into(), &p2aindex_to_p2abytes_mmap)? + .map(|v| AddressBytes::from(v.into_inner())), + OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => { + unreachable!() + } + }; let prev_addressbytes = prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?; @@ -600,6 +647,9 @@ impl Indexer { let mut txindex_to_tx_and_txid: BTreeMap = BTreeMap::default(); + let mut txindex_to_txid_iter = vecs + .txindex_to_txid.iter(); + txid_prefix_to_txid_and_block_txindex_and_prev_txindex .into_iter() .try_for_each( @@ -626,9 +676,8 @@ impl Indexer { let len = vecs.txindex_to_txid.len(); // Ok if `get` is not par as should happen only twice - let prev_txid = vecs - .txindex_to_txid - .get(prev_txindex)? + let prev_txid = txindex_to_txid_iter + .get(prev_txindex) .context("To have txid for txindex") .inspect_err(|_| { dbg!(txindex, len); diff --git a/crates/brk_indexer/src/stores/mod.rs b/crates/brk_indexer/src/stores/mod.rs index cfafe9785..777c0ba21 100644 --- a/crates/brk_indexer/src/stores/mod.rs +++ b/crates/brk_indexer/src/stores/mod.rs @@ -96,12 +96,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2pk65index - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { - while let Some(typedbytes) = vecs - .p2pk65index_to_p2pk65bytes - .get(index)? + let mut p2pk65index_to_p2pk65bytes_iter = vecs.p2pk65index_to_p2pk65bytes.iter(); + + while let Some(typedbytes) = p2pk65index_to_p2pk65bytes_iter + .get(index) .map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); @@ -113,12 +115,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2pk33index - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { - while let Some(typedbytes) = vecs - .p2pk33index_to_p2pk33bytes - .get(index)? + let mut p2pk33index_to_p2pk33bytes_iter = vecs.p2pk33index_to_p2pk33bytes.iter(); + + while let Some(typedbytes) = p2pk33index_to_p2pk33bytes_iter + .get(index) .map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); @@ -130,12 +134,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2pkhindex - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { - while let Some(typedbytes) = vecs - .p2pkhindex_to_p2pkhbytes - .get(index)? + let mut p2pkhindex_to_p2pkhbytes_iter = vecs.p2pkhindex_to_p2pkhbytes.iter(); + + while let Some(typedbytes) = p2pkhindex_to_p2pkhbytes_iter + .get(index) .map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); @@ -147,12 +153,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2shindex - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { - while let Some(typedbytes) = vecs - .p2shindex_to_p2shbytes - .get(index)? + let mut p2shindex_to_p2shbytes_iter = vecs.p2shindex_to_p2shbytes.iter(); + + while let Some(typedbytes) = p2shindex_to_p2shbytes_iter + .get(index) .map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); @@ -164,12 +172,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2trindex - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { - while let Some(typedbytes) = vecs - .p2trindex_to_p2trbytes - .get(index)? + let mut p2trindex_to_p2trbytes_iter = vecs.p2trindex_to_p2trbytes.iter(); + + while let Some(typedbytes) = p2trindex_to_p2trbytes_iter + .get(index) .map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); @@ -181,12 +191,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2wpkhindex - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { - while let Some(typedbytes) = vecs - .p2wpkhindex_to_p2wpkhbytes - .get(index)? + let mut p2wpkhindex_to_p2wpkhbytes_iter = vecs.p2wpkhindex_to_p2wpkhbytes.iter(); + + while let Some(typedbytes) = p2wpkhindex_to_p2wpkhbytes_iter + .get(index) .map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); @@ -198,12 +210,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2wshindex - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { - while let Some(typedbytes) = vecs - .p2wshindex_to_p2wshbytes - .get(index)? + let mut p2wshindex_to_p2wshbytes_iter = vecs.p2wshindex_to_p2wshbytes.iter(); + + while let Some(typedbytes) = p2wshindex_to_p2wshbytes_iter + .get(index) .map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); @@ -215,11 +229,14 @@ impl Stores { if let Some(mut index) = vecs .height_to_first_p2aindex - .get(starting_indexes.height)? + .iter() + .get(starting_indexes.height) .map(Value::into_inner) { + let mut p2aindex_to_p2abytes_iter = vecs.p2aindex_to_p2abytes.iter(); + while let Some(typedbytes) = - vecs.p2aindex_to_p2abytes.get(index)?.map(Value::into_inner) + p2aindex_to_p2abytes_iter.get(index).map(Value::into_inner) { let bytes = AddressBytes::from(typedbytes); let hash = AddressBytesHash::from((&bytes, OutputType::P2A)); diff --git a/crates/brk_indexer/src/vecs/base.rs b/crates/brk_indexer/src/vecs/base.rs index ee925f9ec..464897c77 100644 --- a/crates/brk_indexer/src/vecs/base.rs +++ b/crates/brk_indexer/src/vecs/base.rs @@ -5,7 +5,7 @@ use std::{ }; use brk_vec::{ - Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, + Compressed, DynamicVec, Error, GenericVec, Mmap, Result, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, }; @@ -31,19 +31,15 @@ where version: Version, compressed: Compressed, ) -> brk_vec::Result { - let mut inner = StoredVec::forced_import(path, version, compressed)?; - - inner.enable_large_cache_if_needed(); - Ok(Self { height: Height::try_from(Self::path_height_(path).as_path()).ok(), - inner, + inner: StoredVec::forced_import(path, version, compressed)?, }) } #[inline] - pub fn get(&self, index: I) -> Result>> { - self.inner.get(index) + pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { + self.inner.get_or_read(index, mmap) } #[inline] diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 38c9efe5f..5dbdb88f7 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -79,7 +79,7 @@ impl Vecs { height_to_blockhash: IndexedVec::forced_import( &path.join("height_to_blockhash"), Version::ZERO, - compressed, + Compressed::NO, )?, height_to_difficulty: IndexedVec::forced_import( &path.join("height_to_difficulty"), @@ -204,7 +204,7 @@ impl Vecs { p2aindex_to_p2abytes: IndexedVec::forced_import( &path.join("p2aindex_to_p2abytes"), Version::ZERO, - compressed, + Compressed::NO, )?, p2msindex_to_txindex: IndexedVec::forced_import( &path.join("p2msindex_to_txindex"), @@ -214,37 +214,37 @@ impl Vecs { p2pk33index_to_p2pk33bytes: IndexedVec::forced_import( &path.join("p2pk33index_to_p2pk33bytes"), Version::ZERO, - compressed, + Compressed::NO, )?, p2pk65index_to_p2pk65bytes: IndexedVec::forced_import( &path.join("p2pk65index_to_p2pk65bytes"), Version::ZERO, - compressed, + Compressed::NO, )?, p2pkhindex_to_p2pkhbytes: IndexedVec::forced_import( &path.join("p2pkhindex_to_p2pkhbytes"), Version::ZERO, - compressed, + Compressed::NO, )?, p2shindex_to_p2shbytes: IndexedVec::forced_import( &path.join("p2shindex_to_p2shbytes"), Version::ZERO, - compressed, + Compressed::NO, )?, p2trindex_to_p2trbytes: IndexedVec::forced_import( &path.join("p2trindex_to_p2trbytes"), Version::ZERO, - compressed, + Compressed::NO, )?, p2wpkhindex_to_p2wpkhbytes: IndexedVec::forced_import( &path.join("p2wpkhindex_to_p2wpkhbytes"), Version::ZERO, - compressed, + Compressed::NO, )?, p2wshindex_to_p2wshbytes: IndexedVec::forced_import( &path.join("p2wshindex_to_p2wshbytes"), Version::ZERO, - compressed, + Compressed::NO, )?, txindex_to_base_size: IndexedVec::forced_import( &path.join("txindex_to_base_size"), @@ -259,7 +259,7 @@ impl Vecs { txindex_to_first_outputindex: IndexedVec::forced_import( &path.join("txindex_to_first_outputindex"), Version::ZERO, - compressed, + Compressed::NO, )?, txindex_to_is_explicitly_rbf: IndexedVec::forced_import( &path.join("txindex_to_is_explicitly_rbf"), @@ -279,7 +279,7 @@ impl Vecs { txindex_to_txid: IndexedVec::forced_import( &path.join("txindex_to_txid"), Version::ZERO, - compressed, + Compressed::NO, )?, txindex_to_txversion: IndexedVec::forced_import( &path.join("txindex_to_txversion"), @@ -408,50 +408,6 @@ impl Vecs { Ok(()) } - pub fn get_addressbytes( - &self, - outputtype: OutputType, - outputtypeindex: OutputTypeIndex, - ) -> brk_vec::Result> { - Ok(match outputtype { - OutputType::P2PK65 => self - .p2pk65index_to_p2pk65bytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2PK33 => self - .p2pk33index_to_p2pk33bytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2PKH => self - .p2pkhindex_to_p2pkhbytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2SH => self - .p2shindex_to_p2shbytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2WPKH => self - .p2wpkhindex_to_p2wpkhbytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2WSH => self - .p2wshindex_to_p2wshbytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2TR => self - .p2trindex_to_p2trbytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::P2A => self - .p2aindex_to_p2abytes - .get(outputtypeindex.into())? - .map(|v| AddressBytes::from(v.into_inner())), - OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => { - unreachable!() - } - }) - } - pub fn push_bytes_if_needed( &mut self, index: OutputTypeIndex, diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 329bc2be6..73af84e5d 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -21,7 +21,7 @@ color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -oxc = { version = "0.66.0", features = ["codegen", "minifier"] } +oxc = { version = "0.67.0", features = ["codegen", "minifier"] } serde = { workspace = true } tokio = { version = "1.44.2", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full", "trace"] } diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index 5442dbae7..500a2a6b0 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -15,9 +15,11 @@ fn main() -> Result<(), Box> { (0..21_u32).for_each(|v| { vec.push(v); }); - dbg!(vec.get(0)?); - dbg!(vec.get(20)?); - dbg!(vec.get(21)?); + + let mut iter = vec.iter(); + dbg!(iter.get(0)); + dbg!(iter.get(20)); + dbg!(iter.get(21)); vec.flush()?; } @@ -25,21 +27,25 @@ fn main() -> Result<(), Box> { { let mut vec: StoredVec = StoredVec::forced_import(Path::new("./vec"), version, compressed)?; + let mut iter = vec.iter(); - dbg!(vec.get(0)?); - dbg!(vec.get(0)?); - dbg!(vec.get(1)?); - dbg!(vec.get(2)?); - dbg!(vec.get(20)?); - dbg!(vec.get(20)?); - dbg!(vec.get(0)?); + dbg!(iter.get(0)); + dbg!(iter.get(0)); + dbg!(iter.get(1)); + dbg!(iter.get(2)); + dbg!(iter.get(20)); + dbg!(iter.get(20)); + dbg!(iter.get(0)); vec.push(21); vec.push(22); - dbg!(vec.get(20)?); - dbg!(vec.get(21)?); - dbg!(vec.get(22)?); - dbg!(vec.get(23)?); + + let mut iter = vec.iter(); + + dbg!(iter.get(20)); + dbg!(iter.get(21)); + dbg!(iter.get(22)); + dbg!(iter.get(23)); vec.flush()?; } @@ -47,19 +53,20 @@ fn main() -> Result<(), Box> { { let mut vec: StoredVec = StoredVec::forced_import(Path::new("./vec"), version, compressed)?; + let mut iter = vec.iter(); - vec.enable_large_cache_if_needed(); - - dbg!(vec.get(0)?); - dbg!(vec.get(20)?); - dbg!(vec.get(21)?); - dbg!(vec.get(22)?); + dbg!(iter.get(0)); + dbg!(iter.get(20)); + dbg!(iter.get(21)); + dbg!(iter.get(22)); vec.truncate_if_needed(14)?; - dbg!(vec.get(0)?); - dbg!(vec.get(5)?); - dbg!(vec.get(20)?); + let mut iter = vec.iter(); + + dbg!(iter.get(0)); + dbg!(iter.get(5)); + dbg!(iter.get(20)); dbg!(vec.collect_signed_range(Some(-5), None)?); diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index c3118bea3..3729dd09c 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -8,15 +8,12 @@ mod structs; mod traits; mod variants; -use std::{ - path::{Path, PathBuf}, - sync::Arc, -}; +use std::path::{Path, PathBuf}; -use arc_swap::{ArcSwap, Guard}; +use arc_swap::ArcSwap; use axum::response::Response; pub use enums::*; -use memmap2::Mmap; +pub use memmap2::Mmap; pub use structs::*; pub use traits::*; use variants::*; @@ -46,13 +43,6 @@ where } } - pub fn enable_large_cache_if_needed(&mut self) { - match self { - StoredVec::Compressed(v) => v.enable_large_cache(), - Self::Raw(_) => {} - } - } - pub fn iter(&self) -> StoredVecIterator<'_, I, T> { self.into_iter() } @@ -73,10 +63,10 @@ where type T = T; #[inline] - fn get_stored_(&self, index: usize, guard: &Mmap) -> Result> { + fn read_(&self, index: usize, guard: &Mmap) -> Result> { match self { - StoredVec::Raw(v) => v.get_stored_(index, guard), - StoredVec::Compressed(v) => v.get_stored_(index, guard), + StoredVec::Raw(v) => v.read_(index, guard), + StoredVec::Compressed(v) => v.read_(index, guard), } } @@ -88,21 +78,6 @@ where } } - #[inline] - fn guard(&self) -> &Option>> { - match self { - StoredVec::Raw(v) => v.guard(), - StoredVec::Compressed(v) => v.guard(), - } - } - #[inline] - fn mut_guard(&mut self) -> &mut Option>> { - match self { - StoredVec::Raw(v) => v.mut_guard(), - StoredVec::Compressed(v) => v.mut_guard(), - } - } - #[inline] fn stored_len(&self) -> usize { match self { @@ -250,21 +225,21 @@ where { #[inline] pub fn unwrap_get_inner(&mut self, i: I) -> T { - self.get_(i.unwrap_to_usize()).unwrap().1.into_inner() + self.get_(i.unwrap_to_usize()).unwrap().into_inner() } #[inline] pub fn get_inner(&mut self, i: I) -> Option { - self.get_(i.unwrap_to_usize()).map(|(_, v)| v.into_inner()) + self.get_(i.unwrap_to_usize()).map(|v| v.into_inner()) } #[inline] - pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> { + pub fn get(&mut self, i: I) -> Option> { self.get_(i.unwrap_to_usize()) } #[inline] - pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> { + pub fn get_(&mut self, i: usize) -> Option> { match self { Self::Compressed(iter) => iter.get_(i), Self::Raw(iter) => iter.get_(i), diff --git a/crates/brk_vec/src/traits/dynamic.rs b/crates/brk_vec/src/traits/dynamic.rs index 0c193e375..de95b3eff 100644 --- a/crates/brk_vec/src/traits/dynamic.rs +++ b/crates/brk_vec/src/traits/dynamic.rs @@ -1,45 +1,42 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; -use arc_swap::{ArcSwap, Guard}; +use arc_swap::ArcSwap; use memmap2::Mmap; -use crate::{Error, Result, Value}; +use crate::{Result, Value}; use super::{StoredIndex, StoredType}; pub trait DynamicVec: Send + Sync { type I: StoredIndex; type T: StoredType; + const SIZE_OF_T: usize = size_of::(); #[inline] - fn get(&self, index: Self::I) -> Result>> { - self.get_(index.to_usize()?) + fn read(&self, index: Self::I, mmap: &Mmap) -> Result> { + self.read_(index.to_usize()?, mmap) + } + fn read_(&self, index: usize, mmap: &Mmap) -> Result>; + + #[inline] + fn get_or_read(&self, index: Self::I, mmap: &Mmap) -> Result>> { + self.get_or_read_(index.to_usize()?, mmap) } #[inline] - fn get_(&self, index: usize) -> Result>> { - match self.index_to_pushed_index(index) { - Ok(index) => { - if let Some(index) = index { - return Ok(self.pushed().get(index).map(Value::Ref)); - } + fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result>> { + let stored_len = mmap.len() / Self::SIZE_OF_T; + + if index >= stored_len { + let pushed = self.pushed(); + let j = index - stored_len; + if j >= pushed.len() { + return Ok(None); } - Err(Error::IndexTooHigh) => return Ok(None), - Err(Error::IndexTooLow) => {} - Err(error) => return Err(error), + Ok(pushed.get(j).map(Value::Ref)) + } else { + Ok(self.read_(index, mmap)?.map(Value::Owned)) } - - Ok(self - .get_stored_(index.to_usize()?, self.guard().as_ref().unwrap())? - .map(Value::Owned)) } - fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result>; - // fn last(&self) -> Result>> { - // let len = self.len(); - // if len == 0 { - // return Ok(None); - // } - // self.get_(len - 1) - // } #[inline] fn len(&self) -> usize { @@ -52,9 +49,6 @@ pub trait DynamicVec: Send + Sync { fn mmap(&self) -> &ArcSwap; - fn guard(&self) -> &Option>>; - fn mut_guard(&mut self) -> &mut Option>>; - fn stored_len(&self) -> usize; fn pushed(&self) -> &[Self::T]; @@ -67,21 +61,6 @@ pub trait DynamicVec: Send + Sync { fn push(&mut self, value: Self::T) { self.mut_pushed().push(value) } - #[inline] - fn index_to_pushed_index(&self, index: usize) -> Result> { - let stored_len = self.stored_len(); - - if index >= stored_len { - let index = index - stored_len; - if index >= self.pushed_len() { - Err(Error::IndexTooHigh) - } else { - Ok(Some(index)) - } - } else { - Err(Error::IndexTooLow) - } - } fn path(&self) -> &Path; } diff --git a/crates/brk_vec/src/traits/generic.rs b/crates/brk_vec/src/traits/generic.rs index 216703b60..c5960082f 100644 --- a/crates/brk_vec/src/traits/generic.rs +++ b/crates/brk_vec/src/traits/generic.rs @@ -20,8 +20,6 @@ where I: StoredIndex, T: StoredType, { - const SIZE_OF_T: usize = size_of::(); - fn open_file(&self) -> io::Result { Self::open_file_(&self.path_vec()) } @@ -70,12 +68,6 @@ where fn update_mmap(&mut self, file: File) -> Result<()> { let mmap = Self::new_mmap(file)?; self.mmap().store(mmap); - if self.guard().is_some() { - let guard = self.mmap().load(); - self.mut_guard().replace(guard); - } else { - unreachable!("This function shouldn't be called in a cloned instance") - } Ok(()) } diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index f0a1dca18..07f9c7c0f 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -2,7 +2,7 @@ use std::{ fs::{self, File}, mem, path::Path, - sync::{Arc, OnceLock}, + sync::Arc, }; use arc_swap::{ArcSwap, Guard}; @@ -23,7 +23,6 @@ pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB; #[derive(Debug)] pub struct CompressedVec { inner: RawVec, - decoded_pages: Option>>>, pages_meta: Arc>, } @@ -67,33 +66,10 @@ where Ok(Self { inner: RawVec::import(path, version)?, - decoded_pages: None, pages_meta: Arc::new(ArcSwap::new(Arc::new(CompressedPagesMetadata::read(path)?))), }) } - fn cached_get_stored__( - index: usize, - mmap: &Mmap, - stored_len: usize, - decoded_page: &mut Option<(usize, Vec)>, - compressed_pages_meta: &CompressedPagesMetadata, - ) -> Result> { - let page_index = Self::index_to_page_index(index); - - if decoded_page.as_ref().is_none_or(|b| b.0 != page_index) { - let values = Self::decode_page_(stored_len, page_index, mmap, compressed_pages_meta)?; - decoded_page.replace((page_index, values)); - } - - Ok(decoded_page - .as_ref() - .unwrap() - .1 - .get(index % Self::PER_PAGE) - .cloned()) - } - fn decode_page(&self, page_index: usize, mmap: &Mmap) -> Result> { Self::decode_page_(self.stored_len(), page_index, mmap, &self.pages_meta.load()) } @@ -140,40 +116,6 @@ where zstd::encode_all(bytes.as_slice(), DEFAULT_COMPRESSION_LEVEL).unwrap() } - pub fn enable_large_cache(&mut self) { - self.decoded_pages.replace(vec![]); - self.reset_large_cache(); - } - - pub fn disable_large_cache(&mut self) { - self.decoded_pages.take(); - } - - fn reset_large_cache(&mut self) { - let stored_len = self.stored_len(); - - if let Some(pages) = self.decoded_pages.as_mut() { - pages.par_iter_mut().for_each(|lock| { - lock.take(); - }); - - let len = (stored_len as f64 / Self::PER_PAGE as f64).ceil() as usize; - let len = Self::CACHE_LENGTH.min(len); - - if pages.len() != len { - pages.resize_with(len, Default::default); - } - } - } - - pub fn large_cache_len(&self) -> usize { - self.decoded_pages.as_ref().map_or(0, |v| v.len()) - } - - fn reset_caches(&mut self) { - self.reset_large_cache(); - } - #[inline(always)] fn index_to_page_index(index: usize) -> usize { index / Self::PER_PAGE @@ -219,32 +161,9 @@ where type T = T; #[inline] - fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result> { - let cached_start = self - .stored_len() - .checked_sub(Self::CACHE_LENGTH) - .unwrap_or_default(); - - let decoded_index = index % Self::PER_PAGE; - - if index >= cached_start { - let trimmed_index = index - cached_start; - if let Some(decoded_pages) = self.decoded_pages.as_ref() { - let decoded_page = decoded_pages - .get(Self::index_to_page_index(trimmed_index)) - .unwrap(); - - return Ok(decoded_page - .get_or_init(|| { - self.decode_page(Self::index_to_page_index(index), mmap) - .unwrap() - }) - .get(decoded_index) - .cloned()); - } - } - + fn read_(&self, index: usize, mmap: &Mmap) -> Result> { let page_index = Self::index_to_page_index(index); + let decoded_index = index % Self::PER_PAGE; Ok(self .decode_page(page_index, mmap)? @@ -258,14 +177,6 @@ where } #[inline] - fn guard(&self) -> &Option>> { - self.inner.guard() - } - #[inline] - fn mut_guard(&mut self) -> &mut Option>> { - self.inner.mut_guard() - } - fn stored_len(&self) -> usize { Self::stored_len_(&self.pages_meta.load()) } @@ -328,24 +239,16 @@ where let last_page_index = pages_meta.len() - 1; - values = if let Some(values) = self - .decoded_pages - .as_mut() - .and_then(|v| v.last_mut().and_then(|lock| lock.take())) - { - values - } else { - Self::decode_page_( - stored_len, - last_page_index, - self.guard().as_ref().unwrap(), - &pages_meta, - ) - .inspect_err(|_| { - dbg!(last_page_index, &pages_meta); - }) - .unwrap() - }; + values = Self::decode_page_( + stored_len, + last_page_index, + &self.mmap().load(), + &pages_meta, + ) + .inspect_err(|_| { + dbg!(last_page_index, &pages_meta); + }) + .unwrap(); truncate_at.replace(pages_meta.pop().unwrap().start); starting_page_index = last_page_index; @@ -394,8 +297,6 @@ where self.pages_meta.store(Arc::new(pages_meta)); - self.reset_caches(); - Ok(()) } @@ -404,7 +305,6 @@ where pages_meta.truncate(0); pages_meta.write()?; self.pages_meta.store(Arc::new(pages_meta)); - self.reset_caches(); self.file_truncate_and_write_all(0, &[]) } @@ -424,9 +324,7 @@ where let page_index = Self::index_to_page_index(index); - let guard = self.guard().as_ref().unwrap(); - - let values = self.decode_page(page_index, guard)?; + let values = self.decode_page(page_index, &self.mmap().load())?; let mut buf = vec![]; let mut page = pages_meta.truncate(page_index).unwrap(); @@ -452,8 +350,6 @@ where self.file_truncate_and_write_all(len, &buf)?; - self.reset_caches(); - Ok(()) } @@ -471,7 +367,6 @@ where fn clone(&self) -> Self { Self { inner: self.inner.clone(), - decoded_pages: None, pages_meta: self.pages_meta.clone(), } } @@ -493,6 +388,9 @@ where I: StoredIndex, T: StoredType, { + const SIZE_OF_T: usize = size_of::(); + const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; + #[inline] pub fn set(&mut self, i: I) -> &mut Self { self.index = i.unwrap_to_usize(); @@ -505,14 +403,14 @@ where } #[inline] - pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> { - self.set(i).next() + pub fn get(&mut self, i: I) -> Option> { + self.set(i).next().map(|(_, v)| v) } #[inline] - pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> { + pub fn get_(&mut self, i: usize) -> Option> { self.set_(i); - self.next() + self.next().map(|(_, v)| v) } } @@ -537,15 +435,25 @@ where .get(j) .map(|v| (I::from(i), Value::Ref(v))) } else { - CompressedVec::::cached_get_stored__( - i, - mmap, - stored_len, - &mut self.decoded_page, - &self.pages_meta, - ) - .unwrap() - .map(|v| (I::from(i), Value::Owned(v))) + let page_index = i / Self::PER_PAGE; + + if self.decoded_page.as_ref().is_none_or(|b| b.0 != page_index) { + let values = CompressedVec::::decode_page_( + stored_len, + page_index, + mmap, + &self.pages_meta, + ) + .unwrap(); + self.decoded_page.replace((page_index, values)); + } + + self.decoded_page + .as_ref() + .unwrap() + .1 + .get(i % Self::PER_PAGE) + .map(|v| (I::from(i), Value::Owned(v.clone()))) }; self.index += 1; @@ -562,8 +470,9 @@ where if len == 0 { return None; } - self.get_(len - 1) - .map(|(i, v)| (i, Value::Owned(v.into_inner()))) + let i = len - 1; + self.get_(i) + .map(|v| (I::from(i), Value::Owned(v.into_inner()))) } } diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 98f8fcf51..49a270e9c 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -20,7 +20,6 @@ pub struct RawVec { pathbuf: PathBuf, // Consider Arc>> for dataraces when reorg ? mmap: Arc>, - guard: Option>>, pushed: Vec, phantom: PhantomData, } @@ -51,11 +50,9 @@ where let file = Self::open_file_(Self::path_vec_(path).as_path())?; let mmap = Arc::new(ArcSwap::new(Self::new_mmap(file)?)); - let guard = Some(mmap.load()); Ok(Self { mmap, - guard, version, pathbuf: path.to_owned(), pushed: vec![], @@ -90,7 +87,7 @@ where type T = T; #[inline] - fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result> { + fn read_(&self, index: usize, mmap: &Mmap) -> Result> { let index = index * Self::SIZE_OF_T; let slice = &mmap[index..(index + Self::SIZE_OF_T)]; Self::T::try_read_from_bytes(slice) @@ -103,22 +100,9 @@ where &self.mmap } - #[inline] - fn guard(&self) -> &Option>> { - &self.guard - } - #[inline] - fn mut_guard(&mut self) -> &mut Option>> { - &mut self.guard - } - #[inline] fn stored_len(&self) -> usize { - if let Some(guard) = self.guard() { - guard.len() / Self::SIZE_OF_T - } else { - self.mmap.load().len() / Self::SIZE_OF_T - } + self.mmap.load().len() / Self::SIZE_OF_T } #[inline] @@ -218,9 +202,7 @@ where Self { version: self.version, pathbuf: self.pathbuf.clone(), - // Consider Arc>> for dataraces when reorg ? mmap: self.mmap.clone(), - guard: None, pushed: vec![], phantom: PhantomData, } @@ -239,8 +221,6 @@ where I: StoredIndex, T: StoredType, { - const SIZE_OF_T: usize = size_of::(); - #[inline] pub fn set(&mut self, i: I) -> &mut Self { self.index = i.unwrap_to_usize(); @@ -253,14 +233,14 @@ where } #[inline] - pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> { - self.set(i).next() + pub fn get(&mut self, i: I) -> Option> { + self.set(i).next().map(|(_, v)| v) } #[inline] - pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> { + pub fn get_(&mut self, i: usize) -> Option> { self.set_(i); - self.next() + self.next().map(|(_, v)| v) } } @@ -272,28 +252,21 @@ where type Item = (I, Value<'a, T>); fn next(&mut self) -> Option { let mmap = &self.guard; - let vec = self.vec; - let i = self.index; + let index = self.index; - let stored_len = mmap.len() / Self::SIZE_OF_T; + let opt = self + .vec + .get_or_read_(index, mmap) + .unwrap() + .map(|v| (I::from(index), v)); - let result = if i >= stored_len { - let j = i - stored_len; - if j >= vec.pushed_len() { - return None; - } - vec.pushed().get(j).map(|v| (I::from(i), Value::Ref(v))) - } else { - vec.get_stored_(i, mmap) - .unwrap() - .map(|v| (I::from(i), Value::Owned(v))) - }; + if opt.is_some() { + self.index += 1; + } - self.index += 1; - result + opt } - #[inline] fn last(mut self) -> Option where Self: Sized, @@ -302,8 +275,9 @@ where if len == 0 { return None; } - self.get_(len - 1) - .map(|(i, v)| (i, Value::Owned(v.into_inner()))) + let i = len - 1; + self.get_(i) + .map(|v| (I::from(i), Value::Owned(v.into_inner()))) } } diff --git a/update.sh b/update.sh index d6951dfc1..0d75c46c1 100755 --- a/update.sh +++ b/update.sh @@ -4,4 +4,4 @@ cargo clean rustup update cargo upgrade --incompatible cargo update -cargo build --all-targets +cargo build -r --all-targets