vec: caching only in iter

This commit is contained in:
nym21
2025-04-30 18:29:18 +02:00
parent 664b125ce2
commit 700352ec45
22 changed files with 411 additions and 573 deletions
Generated
+92 -86
View File
@@ -453,7 +453,7 @@ dependencies = [
"rlimit",
"serde",
"serde_bytes",
"zerocopy 0.8.24",
"zerocopy 0.8.25",
]
[[package]]
@@ -494,7 +494,7 @@ dependencies = [
"fjall",
"log",
"rayon",
"zerocopy 0.8.24",
"zerocopy 0.8.25",
]
[[package]]
@@ -519,7 +519,7 @@ dependencies = [
"rayon",
"serde",
"serde_json",
"zerocopy 0.8.24",
"zerocopy 0.8.25",
]
[[package]]
@@ -574,7 +574,7 @@ dependencies = [
"rayon",
"serde",
"serde_json",
"zerocopy 0.8.24",
"zerocopy 0.8.25",
"zstd",
]
@@ -685,9 +685,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.40"
version = "0.4.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
dependencies = [
"android-tzdata",
"iana-time-zone",
@@ -737,7 +737,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -957,7 +957,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -968,7 +968,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -1009,7 +1009,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -1055,7 +1055,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -1121,9 +1121,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "fjall"
version = "2.9.0"
version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "958511f67d1f80e6bff9ffac05c626bb340d4602ca6ea5617d9901c218c894f0"
checksum = "b929b3db7be7d7b4d4df67fb016fc446b8f57507b48a82e69d2a30610e460f28"
dependencies = [
"byteorder",
"byteview",
@@ -1513,7 +1513,7 @@ checksum = "199b7932d97e325aff3a7030e141eafe7f2c6268e1d1b24859b753a627f45254"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -1599,9 +1599,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "lsm-tree"
version = "2.9.0"
version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87d58bdef2dcbf50fce9f343265bdbd7fb08a458d241eb837ce426be22d674b4"
checksum = "d0d03b764a7e3009cc4d314bfce42ce28b4a2c458fc7149b57817cbed7898f43"
dependencies = [
"byteorder",
"crossbeam-skiplist",
@@ -1805,9 +1805,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
[[package]]
name = "oxc"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "409ba57c4752b7fab9609af78790aadeb541316933d1525484dad877988622f6"
checksum = "6f5cf75f3e465c16c27fe188143b2263b74b0729b3cf2044cd946452ce87ee93"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1843,14 +1843,14 @@ checksum = "4c0c893f53900e3fe01eca3d6d3b54085573c3e48fe25af9d57dd94ef600dcd3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
name = "oxc_allocator"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43fe5f8d34c351e3c90b512046dbfff2ff87c8a1927ebe6945f1cdefcc3219d0"
checksum = "8d5e92c9020da9b683bcf06bec4895e618e5d86a290c6e8a805faa7fde701116"
dependencies = [
"allocator-api2",
"bumpalo",
@@ -1862,9 +1862,9 @@ dependencies = [
[[package]]
name = "oxc_ast"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ad2e9c3fb4b409b14fd791fbbf6ced9a81363364beedd48059bbb8f42bf4987"
checksum = "59d46e3425ff92b7edee4c9d299aff471b51fd813780d006b645243e7f522c3f"
dependencies = [
"bitflags",
"cow-utils",
@@ -1879,20 +1879,20 @@ dependencies = [
[[package]]
name = "oxc_ast_macros"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578b69ed39d814cb8396674e17a0f94f1c02bc25d226b265c2913bd6e84afffb"
checksum = "b1be1f451540565035dec927705c83826ca15e96d87d0352395c5e10d4822369"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
name = "oxc_ast_visit"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f437ca959bf931d5facbc6df77c7491a4aec32df5b9f647ffa19c52055a7dc56"
checksum = "7db4c6ac94f2c157a9dd96d5715a656f049653c2088f8c241aea55bd9789b811"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1902,9 +1902,9 @@ dependencies = [
[[package]]
name = "oxc_cfg"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccecf0bce4034ebf17e7edb6f214fa0e6f8d134d1a3fb98408d6bcbef024a3a1"
checksum = "c5bf065a22ae9438554b88b664974be1bcb1060e997bb4ea6be27831af48badb"
dependencies = [
"bitflags",
"itertools",
@@ -1917,9 +1917,9 @@ dependencies = [
[[package]]
name = "oxc_codegen"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3361ed40dd6b4b6a7a4b44c175aa6beea68eefed68bc32a28684a0d1a00bc39"
checksum = "165f1993b33d4c07ec8e90c5522c6afe8985a463eca8545ac1c147863a86d472"
dependencies = [
"bitflags",
"cow-utils",
@@ -1938,15 +1938,15 @@ dependencies = [
[[package]]
name = "oxc_data_structures"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d2d64d2e627796f73e37ccb51f1f87cd9de612245ef3f790051855587d3734e"
checksum = "5dc6d72677e31673f912d567f7d46d6390844a6ceabbc3bd4ef4a99b04178beb"
[[package]]
name = "oxc_diagnostics"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96ece1486240cfeec516eaaa7332c96decb3a756d72e504bb63393507996c39d"
checksum = "5f53beee5473b6929be652ddfed20d3db3e6f96aea57f76a5c7f53ee1e7b458d"
dependencies = [
"cow-utils",
"oxc-miette",
@@ -1954,9 +1954,9 @@ dependencies = [
[[package]]
name = "oxc_ecmascript"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d7d5ddf1cd783ed1a930ac6eedd4f73c6a5359beaabad7ee8333e366816291c"
checksum = "e6674bfaf4ffaa4723cf665e994b630a13cfdc641d2d3f208a010a9ec2bed27c"
dependencies = [
"cow-utils",
"num-bigint",
@@ -1968,9 +1968,9 @@ dependencies = [
[[package]]
name = "oxc_estree"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62de94b10df7ab86a379f01b75076f6a762fc91a001678108318203fc0805ad9"
checksum = "ce1d1d67d2e951ee52e2c9d6e2995be7d54523574396d9c87423092ccb0f6b3c"
[[package]]
name = "oxc_index"
@@ -1980,9 +1980,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
[[package]]
name = "oxc_mangler"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d691c6609bca852ae1e0571de564de3b4d110a46c9877aa029e9dba56b76d1c"
checksum = "6da1b7049a4a0439401b6ae72f1375a0e66a38e4461fb235912cfe67331ce596"
dependencies = [
"fixedbitset",
"itertools",
@@ -1997,9 +1997,9 @@ dependencies = [
[[package]]
name = "oxc_minifier"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "294aa5790cc0beae7018d99dc5b389d373725ef7ea7ecef5d0ed18794b3414b3"
checksum = "c5bae5ecc9c768d15821994a8445e0e6aff763a9bb19a5635d34cab40423366f"
dependencies = [
"cow-utils",
"oxc_allocator",
@@ -2019,9 +2019,9 @@ dependencies = [
[[package]]
name = "oxc_parser"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b7defa49cc20f9e3c346674b06752cacaacce053cde69a04f0f85f31e00820f"
checksum = "67fd747c632d17d36fe496a514c4c87cb2b150f7358a1adb2c01d9d9018e8aa3"
dependencies = [
"bitflags",
"cow-utils",
@@ -2042,9 +2042,9 @@ dependencies = [
[[package]]
name = "oxc_regular_expression"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14d37e8585630a167c4a2b121dcdab52bf13b71a6bd64260efe797956bd4f6d0"
checksum = "298174d22ce142abd379f6a0037d1f46b45e8b2beda894b636c18d7f7dc7645e"
dependencies = [
"oxc_allocator",
"oxc_ast_macros",
@@ -2058,9 +2058,9 @@ dependencies = [
[[package]]
name = "oxc_semantic"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "895130e4f197b77d9b54deffccc2ce8a9d977435c022a49ddecb1f467b946854"
checksum = "f446d2505831faebbe32679dfd3ea5ff2ebafce34ec1cace54541afd7e8102b4"
dependencies = [
"itertools",
"oxc_allocator",
@@ -2094,9 +2094,9 @@ dependencies = [
[[package]]
name = "oxc_span"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8350c982e6284f41668ebd3c714de22280a19f617c3cb6117bbc4e16afc6b13a"
checksum = "75b2d153db7fa3f82316d64f23da65588eabc7763e961c3fe9b4d6224fe5ae36"
dependencies = [
"compact_str",
"oxc-miette",
@@ -2107,9 +2107,9 @@ dependencies = [
[[package]]
name = "oxc_syntax"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e85d9197e08e176d6f58a3885d125738c46a69145410e82b3a32f676ddd5546"
checksum = "faf67148b2cef2b0f8df792e3bbda51957e4c39182e800b0370c60a98b682d1a"
dependencies = [
"bitflags",
"cow-utils",
@@ -2128,11 +2128,10 @@ dependencies = [
[[package]]
name = "oxc_traverse"
version = "0.66.0"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac43e668dcd3a0b17cf4f889feac5321e56430a0550205df2e85b3e9314f6caa"
checksum = "91b56ff860c0a85863cde254924ec2c67da24dbb7b5dda999cab4ae18cbfd245"
dependencies = [
"compact_str",
"itoa",
"oxc_allocator",
"oxc_ast",
@@ -2255,7 +2254,7 @@ dependencies = [
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -2312,7 +2311,7 @@ version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy 0.8.24",
"zerocopy 0.8.25",
]
[[package]]
@@ -2334,7 +2333,7 @@ dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -2628,7 +2627,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -2701,7 +2700,7 @@ dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -2816,9 +2815,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.100"
version = "2.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
dependencies = [
"proc-macro2",
"quote",
@@ -2852,7 +2851,7 @@ dependencies = [
"proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -2905,7 +2904,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -2975,7 +2974,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -2993,9 +2992,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.8.20"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae"
dependencies = [
"serde",
"serde_spanned",
@@ -3005,26 +3004,33 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.8"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.24"
version = "0.22.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e"
dependencies = [
"indexmap 2.9.0",
"serde",
"serde_spanned",
"toml_datetime",
"toml_write",
"winnow",
]
[[package]]
name = "toml_write"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076"
[[package]]
name = "tower"
version = "0.5.2"
@@ -3093,7 +3099,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -3247,7 +3253,7 @@ dependencies = [
"log",
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
"wasm-bindgen-shared",
]
@@ -3269,7 +3275,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -3310,7 +3316,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -3321,7 +3327,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -3474,11 +3480,11 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.24"
version = "0.8.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"
dependencies = [
"zerocopy-derive 0.8.24",
"zerocopy-derive 0.8.25",
]
[[package]]
@@ -3489,18 +3495,18 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.24"
version = "0.8.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
@@ -3520,7 +3526,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"syn 2.0.101",
]
[[package]]
+2 -2
View File
@@ -34,7 +34,7 @@ byteview = "0.6.1"
clap = { version = "4.5.37", features = ["derive", "string"] }
color-eyre = "0.6.3"
derive_deref = "1.1.1"
fjall = "2.9.0"
fjall = "2.10.0"
jiff = "0.2.10"
log = { version = "0.4.27" }
minreq = { version = "2.13.4", features = ["https", "serde_json"] }
@@ -42,7 +42,7 @@ rayon = "1.10.0"
serde = { version = "1.0.219", features = ["derive"] }
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
tabled = "0.19.0"
zerocopy = { version = "0.8.24", features = ["derive"] }
zerocopy = { version = "0.8.25", features = ["derive"] }
[workspace.metadata.release]
shared-version = true
+1 -1
View File
@@ -21,7 +21,7 @@ color-eyre = { workspace = true }
log = { workspace = true }
serde = { workspace = true }
tabled = { workspace = true }
toml = "0.8.20"
toml = "0.8.22"
[[bin]]
name = "brk"
@@ -209,14 +209,14 @@ impl Vecs {
self.difficultyepoch_to_timestamp.compute_transform(
starting_indexes.difficultyepoch,
indexes.difficultyepoch_to_first_height.vec(),
|(i, h, ..)| (i, height_to_timestamp_iter.get(h).unwrap().1.into_inner()),
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
exit,
)?;
self.halvingepoch_to_timestamp.compute_transform(
starting_indexes.halvingepoch,
indexes.halvingepoch_to_first_height.vec(),
|(i, h, ..)| (i, height_to_timestamp_iter.get(h).unwrap().1.into_inner()),
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
exit,
)?;
@@ -186,12 +186,12 @@ where
.try_for_each(|(i, first_index)| -> Result<()> {
let first_index = first_index.into_inner();
let count_index = count_indexes_iter.get(i).unwrap().1.into_inner();
let count_index = count_indexes_iter.unwrap_get_inner(i);
if let Some(first) = self.first.as_mut() {
let f = source_iter
.get(first_index)
.map_or(T::from(0_usize), |f| f.1.into_inner());
.get_inner(first_index)
.unwrap_or_else(|| T::from(0_usize));
first.forced_push_at(index, f, exit)?;
}
@@ -201,15 +201,13 @@ where
panic!("should compute last if count can be 0")
}
let last_index = first_index + (count_index - 1);
let v = source_iter
.get(last_index)
.context("to work")
.inspect_err(|_| {
dbg!(first_index, count_index, last_index);
})
.unwrap()
.1
.into_inner();
let v = source_iter.unwrap_get_inner(last_index);
// .context("to work")
// .inspect_err(|_| {
// dbg!(first_index, count_index, last_index);
// })
// .unwrap()
// .into_inner();
last.forced_push_at(index, v, exit)?;
}
@@ -357,7 +355,7 @@ where
.try_for_each(|(i, first_index, ..)| -> Result<()> {
let first_index = first_index.into_inner();
let count_index = count_indexes_iter.get(i).unwrap().1.into_inner();
let count_index = count_indexes_iter.unwrap_get_inner(i);
if let Some(first) = self.first.as_mut() {
let v = source_first_iter
@@ -342,9 +342,8 @@ impl Vecs {
.get_height(
h,
t,
h.decremented().map(|prev_h| {
height_to_timestamp_iter.get(prev_h).unwrap().1.into_inner()
}),
h.decremented()
.map(|prev_h| height_to_timestamp_iter.unwrap_get_inner(prev_h)),
)
.unwrap();
(h, ohlc)
@@ -471,11 +471,7 @@ impl Vecs {
indexer.vecs().height_to_first_txindex.vec(),
indexer.vecs().txindex_to_txid.vec(),
|txindex| {
let v = txindex_to_txversion_iter
.get(txindex)
.unwrap()
.1
.into_inner();
let v = txindex_to_txversion_iter.unwrap_get_inner(txindex);
v == txversion
},
exit,
@@ -499,11 +495,7 @@ impl Vecs {
starting_indexes.txindex,
indexer.vecs().txindex_to_base_size.vec(),
|(txindex, base_size, ..)| {
let total_size = txindex_to_total_size_iter
.get(txindex)
.unwrap()
.1
.into_inner();
let total_size = txindex_to_total_size_iter.unwrap_get_inner(txindex);
// This is the exact definition of a weight unit, as defined by BIP-141 (quote above).
let wu = usize::from(base_size) * 3 + usize::from(total_size);
@@ -533,7 +525,7 @@ impl Vecs {
|(inputindex, outputindex, ..)| {
let value = if outputindex == OutputIndex::COINBASE {
Sats::ZERO
} else if let Some((_, value)) = outputindex_to_value_iter.get(outputindex) {
} else if let Some(value) = outputindex_to_value_iter.get(outputindex) {
value.into_inner()
} else {
dbg!(inputindex, outputindex, inputs_len);
@@ -12,7 +12,6 @@ use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, Version,
};
use color_eyre::eyre::ContextCompat;
use log::info;
const ONE_KIB: usize = 1024;
@@ -223,11 +222,7 @@ where
if prev_i.is_some_and(|prev_i| prev_i == i) {
return Ok(());
}
if self
.inner
.get(i)?
.is_none_or(|old_v| old_v.into_inner() > v)
{
if self.iter().get_inner(i).is_none_or(|old_v| old_v > v) {
self.forced_push_at(i, v, exit)?;
}
prev_i.replace(i);
@@ -350,8 +345,8 @@ where
.iter_at(index)
.try_for_each(|(i, first_index)| {
let end = other_iter
.get(i + 1)
.map(|(_, v)| v.into_inner().unwrap_to_usize())
.get_inner(i + 1)
.map(|v| v.unwrap_to_usize())
.unwrap_or_else(|| other_to_else.len());
let range = first_index.unwrap_to_usize()..end;
@@ -387,14 +382,7 @@ where
self_to_other.iter_at(index).try_for_each(|(i, other)| {
self.forced_push_at(
i,
T::from(
other_to_self_iter
.get(other.into_inner())
.unwrap()
.1
.into_inner()
== i,
),
T::from(other_to_self_iter.unwrap_get_inner(other.into_inner()) == i),
exit,
)
})?;
@@ -424,12 +412,12 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index)| {
let count = *indexes_count_iter.get(i).unwrap().1.into_inner();
let count = *indexes_count_iter.unwrap_get_inner(i);
let first_index = first_index.unwrap_to_usize();
let range = first_index..first_index + count;
let mut sum = T::from(0_usize);
range.into_iter().for_each(|i| {
sum = sum.clone() + source_iter.get(T2::from(i)).unwrap().1.into_inner();
sum = sum.clone() + source_iter.unwrap_get_inner(T2::from(i));
});
self.forced_push_at(i, sum, exit)
})?;
@@ -477,7 +465,7 @@ impl EagerVec<Height, Dollars> {
let mut price_iter = price.iter();
let index = max_from.min(Height::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| {
let dollars = price_iter.get(i).unwrap().1.into_inner();
let dollars = price_iter.unwrap_get_inner(i);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
@@ -503,8 +491,8 @@ impl EagerVec<TxIndex, Dollars> {
let mut price_iter = price.iter();
let index = max_from.min(TxIndex::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin, ..)| {
let height = i_to_height_iter.get(i).unwrap().1.into_inner();
let dollars = price_iter.get(height).unwrap().1.into_inner();
let height = i_to_height_iter.unwrap_get_inner(i);
let dollars = price_iter.unwrap_get_inner(height);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
+1 -1
View File
@@ -24,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
let outputs = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs, false, true)?;
let mut indexer = Indexer::new(outputs, false, false)?;
indexer.import_stores()?;
indexer.import_vecs()?;
+40 -39
View File
@@ -5,7 +5,7 @@ use brk_core::{
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::{Result, StoredIndex, StoredType, Value};
use brk_vec::{StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::{IndexedVec, Stores, Vecs};
@@ -107,115 +107,116 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
})
.unwrap();
vecs.height_to_blockhash.get(*height).map_or(true, |opt| {
opt.is_none_or(|saved_blockhash| {
vecs.height_to_blockhash
.iter()
.get(*height)
.is_none_or(|saved_blockhash| {
let b = &rpc_blockhash != saved_blockhash.as_ref();
if b {
dbg!(rpc_blockhash, saved_blockhash.as_ref());
}
b
})
})
})
.unwrap_or(starting_height);
Ok(Self {
emptyoutputindex: *starting_index(
emptyoutputindex: starting_index(
&vecs.height_to_first_emptyoutputindex,
&vecs.emptyoutputindex_to_txindex,
height,
)?
)
.context("")?,
height,
p2msindex: *starting_index(
p2msindex: starting_index(
&vecs.height_to_first_p2msindex,
&vecs.p2msindex_to_txindex,
height,
)?
)
.context("")?,
opreturnindex: *starting_index(
opreturnindex: starting_index(
&vecs.height_to_first_opreturnindex,
&vecs.opreturnindex_to_txindex,
height,
)?
)
.context("")?,
p2pk33index: *starting_index(
p2pk33index: starting_index(
&vecs.height_to_first_p2pk33index,
&vecs.p2pk33index_to_p2pk33bytes,
height,
)?
)
.context("")?,
p2pk65index: *starting_index(
p2pk65index: starting_index(
&vecs.height_to_first_p2pk65index,
&vecs.p2pk65index_to_p2pk65bytes,
height,
)?
)
.context("")?,
p2pkhindex: *starting_index(
p2pkhindex: starting_index(
&vecs.height_to_first_p2pkhindex,
&vecs.p2pkhindex_to_p2pkhbytes,
height,
)?
)
.context("")?,
p2shindex: *starting_index(
p2shindex: starting_index(
&vecs.height_to_first_p2shindex,
&vecs.p2shindex_to_p2shbytes,
height,
)?
)
.context("")?,
p2trindex: *starting_index(
p2trindex: starting_index(
&vecs.height_to_first_p2trindex,
&vecs.p2trindex_to_p2trbytes,
height,
)?
)
.context("")?,
p2wpkhindex: *starting_index(
p2wpkhindex: starting_index(
&vecs.height_to_first_p2wpkhindex,
&vecs.p2wpkhindex_to_p2wpkhbytes,
height,
)?
)
.context("")?,
p2wshindex: *starting_index(
p2wshindex: starting_index(
&vecs.height_to_first_p2wshindex,
&vecs.p2wshindex_to_p2wshbytes,
height,
)?
)
.context("")?,
p2aindex: *starting_index(
p2aindex: starting_index(
&vecs.height_to_first_p2aindex,
&vecs.p2aindex_to_p2abytes,
height,
)?
)
.context("")?,
txindex: *starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)?
txindex: starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)
.context("")?,
inputindex: *starting_index(
inputindex: starting_index(
&vecs.height_to_first_inputindex,
&vecs.inputindex_to_outputindex,
height,
)?
)
.context("")?,
outputindex: *starting_index(
outputindex: starting_index(
&vecs.height_to_first_outputindex,
&vecs.outputindex_to_value,
height,
)?
)
.context("")?,
unknownoutputindex: *starting_index(
unknownoutputindex: starting_index(
&vecs.height_to_first_unknownoutputindex,
&vecs.unknownoutputindex_to_txindex,
height,
)?
)
.context("")?,
})
}
}
pub fn starting_index<'a, I, T>(
height_to_index: &'a IndexedVec<Height, I>,
index_to_else: &'a IndexedVec<I, T>,
pub fn starting_index<I, T>(
height_to_index: &IndexedVec<Height, I>,
index_to_else: &IndexedVec<I, T>,
starting_height: Height,
) -> Result<Option<Value<'a, I>>>
) -> Option<I>
where
I: StoredType + StoredIndex + From<usize>,
T: StoredType,
@@ -224,8 +225,8 @@ where
.height()
.is_ok_and(|h| h + 1_u32 == starting_height)
{
Ok(Some(Value::Owned(I::from(index_to_else.len()))))
Some(I::from(index_to_else.len()))
} else {
height_to_index.get(starting_height)
height_to_index.iter().get_inner(starting_height)
}
}
+59 -10
View File
@@ -18,7 +18,7 @@ pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_vec::Compressed;
use brk_vec::{Compressed, DynamicVec};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -242,7 +242,11 @@ impl Indexer {
})
});
let input_source_vec_handle = scope.spawn(|| {
let txindex_to_first_outputindex_mmap = vecs
.txindex_to_first_outputindex.vec().mmap().load();
inputs
.into_par_iter()
.enumerate()
@@ -273,13 +277,11 @@ impl Indexer {
let vout = Vout::from(outpoint.vout);
let outputindex = *vecs
.txindex_to_first_outputindex
.get(prev_txindex)?
let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, &txindex_to_first_outputindex_mmap)?
.context("Expect outputindex to not be none")
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
})?
})?.into_inner()
+ vout;
Ok((inputindex, InputSource::PreviousBlock((
@@ -305,6 +307,16 @@ impl Indexer {
});
let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| {
let p2pk65index_to_p2pk65bytes_mmap = vecs
.p2pk65index_to_p2pk65bytes.vec().mmap().load();
let p2pk33index_to_p2pk33bytes_mmap = vecs.p2pk33index_to_p2pk33bytes.vec().mmap().load();
let p2pkhindex_to_p2pkhbytes_mmap = vecs.p2pkhindex_to_p2pkhbytes.vec().mmap().load();
let p2shindex_to_p2shbytes_mmap = vecs.p2shindex_to_p2shbytes.vec().mmap().load();
let p2wpkhindex_to_p2wpkhbytes_mmap = vecs.p2wpkhindex_to_p2wpkhbytes.vec().mmap().load();
let p2wshindex_to_p2wshbytes_mmap = vecs.p2wshindex_to_p2wshbytes.vec().mmap().load();
let p2trindex_to_p2trbytes_mmap = vecs.p2trindex_to_p2trbytes.vec().mmap().load();
let p2aindex_to_p2abytes_mmap = vecs.p2aindex_to_p2abytes.vec().mmap().load();
outputs
.into_par_iter()
.enumerate()
@@ -349,8 +361,43 @@ impl Indexer {
if let Some(Some(outputtypeindex)) = check_collisions.then_some(outputtypeindex_opt) {
let addressbytes = address_bytes_res.as_ref().unwrap();
let prev_addressbytes_opt =
vecs.get_addressbytes(outputtype, outputtypeindex)?;
let prev_addressbytes_opt = match outputtype {
OutputType::P2PK65 => vecs
.p2pk65index_to_p2pk65bytes
.get_or_read(outputtypeindex.into(), &p2pk65index_to_p2pk65bytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PK33 => vecs
.p2pk33index_to_p2pk33bytes
.get_or_read(outputtypeindex.into(), &p2pk33index_to_p2pk33bytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PKH => vecs
.p2pkhindex_to_p2pkhbytes
.get_or_read(outputtypeindex.into(), &p2pkhindex_to_p2pkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2SH => vecs
.p2shindex_to_p2shbytes
.get_or_read(outputtypeindex.into(), &p2shindex_to_p2shbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WPKH => vecs
.p2wpkhindex_to_p2wpkhbytes
.get_or_read(outputtypeindex.into(), &p2wpkhindex_to_p2wpkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WSH => vecs
.p2wshindex_to_p2wshbytes
.get_or_read(outputtypeindex.into(), &p2wshindex_to_p2wshbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2TR => vecs
.p2trindex_to_p2trbytes
.get_or_read(outputtypeindex.into(), &p2trindex_to_p2trbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2A => vecs
.p2aindex_to_p2abytes
.get_or_read(outputtypeindex.into(), &p2aindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
};
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
@@ -600,6 +647,9 @@ impl Indexer {
let mut txindex_to_tx_and_txid: BTreeMap<TxIndex, (&Transaction, Txid)> = BTreeMap::default();
let mut txindex_to_txid_iter = vecs
.txindex_to_txid.iter();
txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.into_iter()
.try_for_each(
@@ -626,9 +676,8 @@ impl Indexer {
let len = vecs.txindex_to_txid.len();
// Ok if `get` is not par as should happen only twice
let prev_txid = vecs
.txindex_to_txid
.get(prev_txindex)?
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.context("To have txid for txindex")
.inspect_err(|_| {
dbg!(txindex, len);
+47 -30
View File
@@ -96,12 +96,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pk65index
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65bytes
.get(index)?
let mut p2pk65index_to_p2pk65bytes_iter = vecs.p2pk65index_to_p2pk65bytes.iter();
while let Some(typedbytes) = p2pk65index_to_p2pk65bytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -113,12 +115,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pk33index
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33bytes
.get(index)?
let mut p2pk33index_to_p2pk33bytes_iter = vecs.p2pk33index_to_p2pk33bytes.iter();
while let Some(typedbytes) = p2pk33index_to_p2pk33bytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -130,12 +134,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pkhindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhbytes
.get(index)?
let mut p2pkhindex_to_p2pkhbytes_iter = vecs.p2pkhindex_to_p2pkhbytes.iter();
while let Some(typedbytes) = p2pkhindex_to_p2pkhbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -147,12 +153,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2shindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2shindex_to_p2shbytes
.get(index)?
let mut p2shindex_to_p2shbytes_iter = vecs.p2shindex_to_p2shbytes.iter();
while let Some(typedbytes) = p2shindex_to_p2shbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -164,12 +172,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2trindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2trindex_to_p2trbytes
.get(index)?
let mut p2trindex_to_p2trbytes_iter = vecs.p2trindex_to_p2trbytes.iter();
while let Some(typedbytes) = p2trindex_to_p2trbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -181,12 +191,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2wpkhindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhbytes
.get(index)?
let mut p2wpkhindex_to_p2wpkhbytes_iter = vecs.p2wpkhindex_to_p2wpkhbytes.iter();
while let Some(typedbytes) = p2wpkhindex_to_p2wpkhbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -198,12 +210,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2wshindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2wshindex_to_p2wshbytes
.get(index)?
let mut p2wshindex_to_p2wshbytes_iter = vecs.p2wshindex_to_p2wshbytes.iter();
while let Some(typedbytes) = p2wshindex_to_p2wshbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -215,11 +229,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2aindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
let mut p2aindex_to_p2abytes_iter = vecs.p2aindex_to_p2abytes.iter();
while let Some(typedbytes) =
vecs.p2aindex_to_p2abytes.get(index)?.map(Value::into_inner)
p2aindex_to_p2abytes_iter.get(index).map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from((&bytes, OutputType::P2A));
+4 -8
View File
@@ -5,7 +5,7 @@ use std::{
};
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
Compressed, DynamicVec, Error, GenericVec, Mmap, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, Version,
};
@@ -31,19 +31,15 @@ where
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
let mut inner = StoredVec::forced_import(path, version, compressed)?;
inner.enable_large_cache_if_needed();
Ok(Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
inner,
inner: StoredVec::forced_import(path, version, compressed)?,
})
}
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
self.inner.get(index)
pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Value<T>>> {
self.inner.get_or_read(index, mmap)
}
#[inline]
+11 -55
View File
@@ -79,7 +79,7 @@ impl Vecs {
height_to_blockhash: IndexedVec::forced_import(
&path.join("height_to_blockhash"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
height_to_difficulty: IndexedVec::forced_import(
&path.join("height_to_difficulty"),
@@ -204,7 +204,7 @@ impl Vecs {
p2aindex_to_p2abytes: IndexedVec::forced_import(
&path.join("p2aindex_to_p2abytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2msindex_to_txindex: IndexedVec::forced_import(
&path.join("p2msindex_to_txindex"),
@@ -214,37 +214,37 @@ impl Vecs {
p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
&path.join("p2pk33index_to_p2pk33bytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2pk65index_to_p2pk65bytes: IndexedVec::forced_import(
&path.join("p2pk65index_to_p2pk65bytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2pkhindex_to_p2pkhbytes: IndexedVec::forced_import(
&path.join("p2pkhindex_to_p2pkhbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2shindex_to_p2shbytes: IndexedVec::forced_import(
&path.join("p2shindex_to_p2shbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2trindex_to_p2trbytes: IndexedVec::forced_import(
&path.join("p2trindex_to_p2trbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2wpkhindex_to_p2wpkhbytes: IndexedVec::forced_import(
&path.join("p2wpkhindex_to_p2wpkhbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2wshindex_to_p2wshbytes: IndexedVec::forced_import(
&path.join("p2wshindex_to_p2wshbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_base_size: IndexedVec::forced_import(
&path.join("txindex_to_base_size"),
@@ -259,7 +259,7 @@ impl Vecs {
txindex_to_first_outputindex: IndexedVec::forced_import(
&path.join("txindex_to_first_outputindex"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_is_explicitly_rbf: IndexedVec::forced_import(
&path.join("txindex_to_is_explicitly_rbf"),
@@ -279,7 +279,7 @@ impl Vecs {
txindex_to_txid: IndexedVec::forced_import(
&path.join("txindex_to_txid"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_txversion: IndexedVec::forced_import(
&path.join("txindex_to_txversion"),
@@ -408,50 +408,6 @@ impl Vecs {
Ok(())
}
pub fn get_addressbytes(
&self,
outputtype: OutputType,
outputtypeindex: OutputTypeIndex,
) -> brk_vec::Result<Option<AddressBytes>> {
Ok(match outputtype {
OutputType::P2PK65 => self
.p2pk65index_to_p2pk65bytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PK33 => self
.p2pk33index_to_p2pk33bytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PKH => self
.p2pkhindex_to_p2pkhbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2SH => self
.p2shindex_to_p2shbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WPKH => self
.p2wpkhindex_to_p2wpkhbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WSH => self
.p2wshindex_to_p2wshbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2TR => self
.p2trindex_to_p2trbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2A => self
.p2aindex_to_p2abytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
})
}
pub fn push_bytes_if_needed(
&mut self,
index: OutputTypeIndex,
+1 -1
View File
@@ -21,7 +21,7 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.66.0", features = ["codegen", "minifier"] }
oxc = { version = "0.67.0", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { version = "1.44.2", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full", "trace"] }
+30 -23
View File
@@ -15,9 +15,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
(0..21_u32).for_each(|v| {
vec.push(v);
});
dbg!(vec.get(0)?);
dbg!(vec.get(20)?);
dbg!(vec.get(21)?);
let mut iter = vec.iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
vec.flush()?;
}
@@ -25,21 +27,25 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StoredVec<usize, u32> =
StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
let mut iter = vec.iter();
dbg!(vec.get(0)?);
dbg!(vec.get(0)?);
dbg!(vec.get(1)?);
dbg!(vec.get(2)?);
dbg!(vec.get(20)?);
dbg!(vec.get(20)?);
dbg!(vec.get(0)?);
dbg!(iter.get(0));
dbg!(iter.get(0));
dbg!(iter.get(1));
dbg!(iter.get(2));
dbg!(iter.get(20));
dbg!(iter.get(20));
dbg!(iter.get(0));
vec.push(21);
vec.push(22);
dbg!(vec.get(20)?);
dbg!(vec.get(21)?);
dbg!(vec.get(22)?);
dbg!(vec.get(23)?);
let mut iter = vec.iter();
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
dbg!(iter.get(23));
vec.flush()?;
}
@@ -47,19 +53,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StoredVec<usize, u32> =
StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
let mut iter = vec.iter();
vec.enable_large_cache_if_needed();
dbg!(vec.get(0)?);
dbg!(vec.get(20)?);
dbg!(vec.get(21)?);
dbg!(vec.get(22)?);
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
vec.truncate_if_needed(14)?;
dbg!(vec.get(0)?);
dbg!(vec.get(5)?);
dbg!(vec.get(20)?);
let mut iter = vec.iter();
dbg!(iter.get(0));
dbg!(iter.get(5));
dbg!(iter.get(20));
dbg!(vec.collect_signed_range(Some(-5), None)?);
+10 -35
View File
@@ -8,15 +8,12 @@ mod structs;
mod traits;
mod variants;
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use std::path::{Path, PathBuf};
use arc_swap::{ArcSwap, Guard};
use arc_swap::ArcSwap;
use axum::response::Response;
pub use enums::*;
use memmap2::Mmap;
pub use memmap2::Mmap;
pub use structs::*;
pub use traits::*;
use variants::*;
@@ -46,13 +43,6 @@ where
}
}
pub fn enable_large_cache_if_needed(&mut self) {
match self {
StoredVec::Compressed(v) => v.enable_large_cache(),
Self::Raw(_) => {}
}
}
pub fn iter(&self) -> StoredVecIterator<'_, I, T> {
self.into_iter()
}
@@ -73,10 +63,10 @@ where
type T = T;
#[inline]
fn get_stored_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
fn read_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
match self {
StoredVec::Raw(v) => v.get_stored_(index, guard),
StoredVec::Compressed(v) => v.get_stored_(index, guard),
StoredVec::Raw(v) => v.read_(index, guard),
StoredVec::Compressed(v) => v.read_(index, guard),
}
}
@@ -88,21 +78,6 @@ where
}
}
#[inline]
fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
match self {
StoredVec::Raw(v) => v.guard(),
StoredVec::Compressed(v) => v.guard(),
}
}
#[inline]
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
match self {
StoredVec::Raw(v) => v.mut_guard(),
StoredVec::Compressed(v) => v.mut_guard(),
}
}
#[inline]
fn stored_len(&self) -> usize {
match self {
@@ -250,21 +225,21 @@ where
{
#[inline]
pub fn unwrap_get_inner(&mut self, i: I) -> T {
self.get_(i.unwrap_to_usize()).unwrap().1.into_inner()
self.get_(i.unwrap_to_usize()).unwrap().into_inner()
}
#[inline]
pub fn get_inner(&mut self, i: I) -> Option<T> {
self.get_(i.unwrap_to_usize()).map(|(_, v)| v.into_inner())
self.get_(i.unwrap_to_usize()).map(|v| v.into_inner())
}
#[inline]
pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> {
pub fn get(&mut self, i: I) -> Option<Value<'_, T>> {
self.get_(i.unwrap_to_usize())
}
#[inline]
pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> {
pub fn get_(&mut self, i: usize) -> Option<Value<'_, T>> {
match self {
Self::Compressed(iter) => iter.get_(i),
Self::Raw(iter) => iter.get_(i),
+23 -44
View File
@@ -1,45 +1,42 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use arc_swap::{ArcSwap, Guard};
use arc_swap::ArcSwap;
use memmap2::Mmap;
use crate::{Error, Result, Value};
use crate::{Result, Value};
use super::{StoredIndex, StoredType};
pub trait DynamicVec: Send + Sync {
type I: StoredIndex;
type T: StoredType;
const SIZE_OF_T: usize = size_of::<Self::T>();
#[inline]
fn get(&self, index: Self::I) -> Result<Option<Value<Self::T>>> {
self.get_(index.to_usize()?)
fn read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Self::T>> {
self.read_(index.to_usize()?, mmap)
}
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
#[inline]
fn get_or_read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
self.get_or_read_(index.to_usize()?, mmap)
}
#[inline]
fn get_(&self, index: usize) -> Result<Option<Value<Self::T>>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
return Ok(self.pushed().get(index).map(Value::Ref));
}
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
let stored_len = mmap.len() / Self::SIZE_OF_T;
if index >= stored_len {
let pushed = self.pushed();
let j = index - stored_len;
if j >= pushed.len() {
return Ok(None);
}
Err(Error::IndexTooHigh) => return Ok(None),
Err(Error::IndexTooLow) => {}
Err(error) => return Err(error),
Ok(pushed.get(j).map(Value::Ref))
} else {
Ok(self.read_(index, mmap)?.map(Value::Owned))
}
Ok(self
.get_stored_(index.to_usize()?, self.guard().as_ref().unwrap())?
.map(Value::Owned))
}
fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
// fn last(&self) -> Result<Option<Value<Self::T>>> {
// let len = self.len();
// if len == 0 {
// return Ok(None);
// }
// self.get_(len - 1)
// }
#[inline]
fn len(&self) -> usize {
@@ -52,9 +49,6 @@ pub trait DynamicVec: Send + Sync {
fn mmap(&self) -> &ArcSwap<Mmap>;
fn guard(&self) -> &Option<Guard<Arc<Mmap>>>;
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>>;
fn stored_len(&self) -> usize;
fn pushed(&self) -> &[Self::T];
@@ -67,21 +61,6 @@ pub trait DynamicVec: Send + Sync {
fn push(&mut self, value: Self::T) {
self.mut_pushed().push(value)
}
#[inline]
fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
let stored_len = self.stored_len();
if index >= stored_len {
let index = index - stored_len;
if index >= self.pushed_len() {
Err(Error::IndexTooHigh)
} else {
Ok(Some(index))
}
} else {
Err(Error::IndexTooLow)
}
}
fn path(&self) -> &Path;
}
-8
View File
@@ -20,8 +20,6 @@ where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF_T: usize = size_of::<Self::T>();
fn open_file(&self) -> io::Result<File> {
Self::open_file_(&self.path_vec())
}
@@ -70,12 +68,6 @@ where
fn update_mmap(&mut self, file: File) -> Result<()> {
let mmap = Self::new_mmap(file)?;
self.mmap().store(mmap);
if self.guard().is_some() {
let guard = self.mmap().load();
self.mut_guard().replace(guard);
} else {
unreachable!("This function shouldn't be called in a cloned instance")
}
Ok(())
}
+43 -134
View File
@@ -2,7 +2,7 @@ use std::{
fs::{self, File},
mem,
path::Path,
sync::{Arc, OnceLock},
sync::Arc,
};
use arc_swap::{ArcSwap, Guard};
@@ -23,7 +23,6 @@ pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
#[derive(Debug)]
pub struct CompressedVec<I, T> {
inner: RawVec<I, T>,
decoded_pages: Option<Vec<OnceLock<Vec<T>>>>,
pages_meta: Arc<ArcSwap<CompressedPagesMetadata>>,
}
@@ -67,33 +66,10 @@ where
Ok(Self {
inner: RawVec::import(path, version)?,
decoded_pages: None,
pages_meta: Arc::new(ArcSwap::new(Arc::new(CompressedPagesMetadata::read(path)?))),
})
}
fn cached_get_stored__(
index: usize,
mmap: &Mmap,
stored_len: usize,
decoded_page: &mut Option<(usize, Vec<T>)>,
compressed_pages_meta: &CompressedPagesMetadata,
) -> Result<Option<T>> {
let page_index = Self::index_to_page_index(index);
if decoded_page.as_ref().is_none_or(|b| b.0 != page_index) {
let values = Self::decode_page_(stored_len, page_index, mmap, compressed_pages_meta)?;
decoded_page.replace((page_index, values));
}
Ok(decoded_page
.as_ref()
.unwrap()
.1
.get(index % Self::PER_PAGE)
.cloned())
}
fn decode_page(&self, page_index: usize, mmap: &Mmap) -> Result<Vec<T>> {
Self::decode_page_(self.stored_len(), page_index, mmap, &self.pages_meta.load())
}
@@ -140,40 +116,6 @@ where
zstd::encode_all(bytes.as_slice(), DEFAULT_COMPRESSION_LEVEL).unwrap()
}
pub fn enable_large_cache(&mut self) {
self.decoded_pages.replace(vec![]);
self.reset_large_cache();
}
pub fn disable_large_cache(&mut self) {
self.decoded_pages.take();
}
fn reset_large_cache(&mut self) {
let stored_len = self.stored_len();
if let Some(pages) = self.decoded_pages.as_mut() {
pages.par_iter_mut().for_each(|lock| {
lock.take();
});
let len = (stored_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
let len = Self::CACHE_LENGTH.min(len);
if pages.len() != len {
pages.resize_with(len, Default::default);
}
}
}
pub fn large_cache_len(&self) -> usize {
self.decoded_pages.as_ref().map_or(0, |v| v.len())
}
fn reset_caches(&mut self) {
self.reset_large_cache();
}
#[inline(always)]
fn index_to_page_index(index: usize) -> usize {
index / Self::PER_PAGE
@@ -219,32 +161,9 @@ where
type T = T;
#[inline]
fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let cached_start = self
.stored_len()
.checked_sub(Self::CACHE_LENGTH)
.unwrap_or_default();
let decoded_index = index % Self::PER_PAGE;
if index >= cached_start {
let trimmed_index = index - cached_start;
if let Some(decoded_pages) = self.decoded_pages.as_ref() {
let decoded_page = decoded_pages
.get(Self::index_to_page_index(trimmed_index))
.unwrap();
return Ok(decoded_page
.get_or_init(|| {
self.decode_page(Self::index_to_page_index(index), mmap)
.unwrap()
})
.get(decoded_index)
.cloned());
}
}
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let page_index = Self::index_to_page_index(index);
let decoded_index = index % Self::PER_PAGE;
Ok(self
.decode_page(page_index, mmap)?
@@ -258,14 +177,6 @@ where
}
#[inline]
fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
self.inner.guard()
}
#[inline]
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
self.inner.mut_guard()
}
fn stored_len(&self) -> usize {
Self::stored_len_(&self.pages_meta.load())
}
@@ -328,24 +239,16 @@ where
let last_page_index = pages_meta.len() - 1;
values = if let Some(values) = self
.decoded_pages
.as_mut()
.and_then(|v| v.last_mut().and_then(|lock| lock.take()))
{
values
} else {
Self::decode_page_(
stored_len,
last_page_index,
self.guard().as_ref().unwrap(),
&pages_meta,
)
.inspect_err(|_| {
dbg!(last_page_index, &pages_meta);
})
.unwrap()
};
values = Self::decode_page_(
stored_len,
last_page_index,
&self.mmap().load(),
&pages_meta,
)
.inspect_err(|_| {
dbg!(last_page_index, &pages_meta);
})
.unwrap();
truncate_at.replace(pages_meta.pop().unwrap().start);
starting_page_index = last_page_index;
@@ -394,8 +297,6 @@ where
self.pages_meta.store(Arc::new(pages_meta));
self.reset_caches();
Ok(())
}
@@ -404,7 +305,6 @@ where
pages_meta.truncate(0);
pages_meta.write()?;
self.pages_meta.store(Arc::new(pages_meta));
self.reset_caches();
self.file_truncate_and_write_all(0, &[])
}
@@ -424,9 +324,7 @@ where
let page_index = Self::index_to_page_index(index);
let guard = self.guard().as_ref().unwrap();
let values = self.decode_page(page_index, guard)?;
let values = self.decode_page(page_index, &self.mmap().load())?;
let mut buf = vec![];
let mut page = pages_meta.truncate(page_index).unwrap();
@@ -452,8 +350,6 @@ where
self.file_truncate_and_write_all(len, &buf)?;
self.reset_caches();
Ok(())
}
@@ -471,7 +367,6 @@ where
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
decoded_pages: None,
pages_meta: self.pages_meta.clone(),
}
}
@@ -493,6 +388,9 @@ where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF_T: usize = size_of::<T>();
const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
#[inline]
pub fn set(&mut self, i: I) -> &mut Self {
self.index = i.unwrap_to_usize();
@@ -505,14 +403,14 @@ where
}
#[inline]
pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> {
self.set(i).next()
pub fn get(&mut self, i: I) -> Option<Value<'_, T>> {
self.set(i).next().map(|(_, v)| v)
}
#[inline]
pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> {
pub fn get_(&mut self, i: usize) -> Option<Value<'_, T>> {
self.set_(i);
self.next()
self.next().map(|(_, v)| v)
}
}
@@ -537,15 +435,25 @@ where
.get(j)
.map(|v| (I::from(i), Value::Ref(v)))
} else {
CompressedVec::<I, T>::cached_get_stored__(
i,
mmap,
stored_len,
&mut self.decoded_page,
&self.pages_meta,
)
.unwrap()
.map(|v| (I::from(i), Value::Owned(v)))
let page_index = i / Self::PER_PAGE;
if self.decoded_page.as_ref().is_none_or(|b| b.0 != page_index) {
let values = CompressedVec::<I, T>::decode_page_(
stored_len,
page_index,
mmap,
&self.pages_meta,
)
.unwrap();
self.decoded_page.replace((page_index, values));
}
self.decoded_page
.as_ref()
.unwrap()
.1
.get(i % Self::PER_PAGE)
.map(|v| (I::from(i), Value::Owned(v.clone())))
};
self.index += 1;
@@ -562,8 +470,9 @@ where
if len == 0 {
return None;
}
self.get_(len - 1)
.map(|(i, v)| (i, Value::Owned(v.into_inner())))
let i = len - 1;
self.get_(i)
.map(|v| (I::from(i), Value::Owned(v.into_inner())))
}
}
+19 -45
View File
@@ -20,7 +20,6 @@ pub struct RawVec<I, T> {
pathbuf: PathBuf,
// Consider Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
mmap: Arc<ArcSwap<Mmap>>,
guard: Option<Guard<Arc<Mmap>>>,
pushed: Vec<T>,
phantom: PhantomData<I>,
}
@@ -51,11 +50,9 @@ where
let file = Self::open_file_(Self::path_vec_(path).as_path())?;
let mmap = Arc::new(ArcSwap::new(Self::new_mmap(file)?));
let guard = Some(mmap.load());
Ok(Self {
mmap,
guard,
version,
pathbuf: path.to_owned(),
pushed: vec![],
@@ -90,7 +87,7 @@ where
type T = T;
#[inline]
fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let index = index * Self::SIZE_OF_T;
let slice = &mmap[index..(index + Self::SIZE_OF_T)];
Self::T::try_read_from_bytes(slice)
@@ -103,22 +100,9 @@ where
&self.mmap
}
#[inline]
fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
&self.guard
}
#[inline]
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
&mut self.guard
}
#[inline]
fn stored_len(&self) -> usize {
if let Some(guard) = self.guard() {
guard.len() / Self::SIZE_OF_T
} else {
self.mmap.load().len() / Self::SIZE_OF_T
}
self.mmap.load().len() / Self::SIZE_OF_T
}
#[inline]
@@ -218,9 +202,7 @@ where
Self {
version: self.version,
pathbuf: self.pathbuf.clone(),
// Consider Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
mmap: self.mmap.clone(),
guard: None,
pushed: vec![],
phantom: PhantomData,
}
@@ -239,8 +221,6 @@ where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF_T: usize = size_of::<T>();
#[inline]
pub fn set(&mut self, i: I) -> &mut Self {
self.index = i.unwrap_to_usize();
@@ -253,14 +233,14 @@ where
}
#[inline]
pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> {
self.set(i).next()
pub fn get(&mut self, i: I) -> Option<Value<'_, T>> {
self.set(i).next().map(|(_, v)| v)
}
#[inline]
pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> {
pub fn get_(&mut self, i: usize) -> Option<Value<'_, T>> {
self.set_(i);
self.next()
self.next().map(|(_, v)| v)
}
}
@@ -272,28 +252,21 @@ where
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let mmap = &self.guard;
let vec = self.vec;
let i = self.index;
let index = self.index;
let stored_len = mmap.len() / Self::SIZE_OF_T;
let opt = self
.vec
.get_or_read_(index, mmap)
.unwrap()
.map(|v| (I::from(index), v));
let result = if i >= stored_len {
let j = i - stored_len;
if j >= vec.pushed_len() {
return None;
}
vec.pushed().get(j).map(|v| (I::from(i), Value::Ref(v)))
} else {
vec.get_stored_(i, mmap)
.unwrap()
.map(|v| (I::from(i), Value::Owned(v)))
};
if opt.is_some() {
self.index += 1;
}
self.index += 1;
result
opt
}
#[inline]
fn last(mut self) -> Option<Self::Item>
where
Self: Sized,
@@ -302,8 +275,9 @@ where
if len == 0 {
return None;
}
self.get_(len - 1)
.map(|(i, v)| (i, Value::Owned(v.into_inner())))
let i = len - 1;
self.get_(i)
.map(|v| (I::from(i), Value::Owned(v.into_inner())))
}
}
+1 -1
View File
@@ -4,4 +4,4 @@ cargo clean
rustup update
cargo upgrade --incompatible
cargo update
cargo build --all-targets
cargo build -r --all-targets