diff --git a/Cargo.lock b/Cargo.lock index 2292998e4..bdcacd37e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,6 +506,7 @@ dependencies = [ name = "brk" version = "0.0.111" dependencies = [ + "brk_bencher", "brk_binder", "brk_bundler", "brk_cli", @@ -527,6 +528,20 @@ dependencies = [ "brk_types", ] +[[package]] +name = "brk_bencher" +version = "0.0.111" +dependencies = [ + "brk_error", +] + +[[package]] +name = "brk_bencher_visualizer" +version = "0.0.111" +dependencies = [ + "plotters", +] + [[package]] name = "brk_binder" version = "0.0.111" @@ -631,8 +646,6 @@ dependencies = [ [[package]] name = "brk_fjall" version = "2.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7d4da994dbf749f9ae892b1f447c7b0adda81ad16098d6bd9c3dd58be3d65b8" dependencies = [ "byteorder", "byteview 0.6.1", @@ -661,6 +674,7 @@ name = "brk_indexer" version = "0.0.111" dependencies = [ "bitcoin", + "brk_bencher", "brk_error", "brk_fjall", "brk_grouper", @@ -673,7 +687,6 @@ dependencies = [ "brk_types", "fjall", "log", - "rand 0.9.2", "rayon", "rustc-hash", "vecdb", @@ -1303,6 +1316,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" @@ -1458,6 +1477,12 @@ dependencies = [ "tracing-error", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.4" @@ -1529,12 +1554,58 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "libc", +] + +[[package]] +name = "core-text" +version = "20.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5" +dependencies = [ + "core-foundation", + "core-graphics", + "foreign-types", + "libc", +] + [[package]] name = "cow-utils" version = "0.1.3" @@ -1772,6 +1843,27 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "dispatch2" version = "0.3.0" @@ -1795,6 +1887,15 @@ dependencies = [ "syn 2.0.109", ] +[[package]] +name = "dlib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" +dependencies = [ + "libloading", +] + [[package]] name = "double-ended-peekable" version = "0.1.0" @@ -1819,6 +1920,18 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dwrote" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b35532432acc8b19ceed096e35dfa088d3ea037fe4f3c085f1f97f33b4d02" +dependencies = [ + "lazy_static", + "libc", + "winapi", + "wio", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1936,6 +2049,15 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filetime" version = "0.2.26" @@ -1962,16 +2084,15 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "fjall" -version = "3.0.0-pre.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf4746eb86124d4cd91564bc01c247b8c2c3ddf2d9fb3701ca16c86fdb5b538" +version = "3.0.0-pre.5" dependencies = [ "byteorder-lite", "byteview 0.8.0", "dashmap", + "flume", "log", "lsm-tree 3.0.0-pre.5", - "std-semaphore", + "lz4_flex", "tempfile", "xxhash-rust", ] @@ -1996,6 +2117,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2014,6 +2150,58 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "font-kit" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7e611d49285d4c4b2e1727b72cf05353558885cc5252f93707b845dfcaf3d3" +dependencies = [ + "bitflags 2.10.0", + "byteorder", + "core-foundation", + "core-graphics", + "core-text", + "dirs", + "dwrote", + "float-ord", + "freetype-sys", + "lazy_static", + "libc", + "log", + "pathfinder_geometry", + "pathfinder_simd", + "walkdir", + "winapi", + "yeslogic-fontconfig-sys", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.109", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2023,6 +2211,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "freetype-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7edc5b9669349acfda99533e9e0bcf26a51862ab43b08ee7745c55d28eb134" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2154,6 +2353,16 @@ dependencies = [ "wasip2", ] +[[package]] +name = "gif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.32.3" @@ -2448,6 +2657,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-traits", + "png", +] + [[package]] name = "indenter" version = "0.3.4" @@ -2576,6 +2799,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + [[package]] name = "js-sys" version = "0.3.82" @@ -2645,6 +2874,16 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libredox" version = "0.1.10" @@ -2719,8 +2958,6 @@ dependencies = [ [[package]] name = "lsm-tree" version = "3.0.0-pre.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1cdfd737b63065d5162e39b581d6c9575990af0669cd200d5031bfd88adf25" dependencies = [ "byteorder-lite", "byteview 0.8.0", @@ -2743,6 +2980,9 @@ name = "lz4_flex" version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] [[package]] name = "matchit" @@ -2987,6 +3227,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "outref" version = "0.5.2" @@ -3552,6 +3798,25 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pathfinder_geometry" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3" +dependencies = [ + "log", + "pathfinder_simd", +] + +[[package]] +name = "pathfinder_simd" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf9027960355bf3afff9841918474a81a5f972ac6d226d518060bba758b5ad57" +dependencies = [ + "rustc_version", +] + [[package]] name = "pco" version = "0.4.7" @@ -3663,6 +3928,65 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "chrono", + "font-kit", + "image", + "lazy_static", + "num-traits", + "pathfinder_geometry", + "plotters-backend", + "plotters-bitmap", + "plotters-svg", + "ttf-parser", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-bitmap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405" +dependencies = [ + "gif", + "image", + "plotters-backend", +] + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "pnp" version = "0.12.5" @@ -3930,6 +4254,17 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 2.0.17", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -4116,6 +4451,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.2" @@ -4252,6 +4596,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16c2f82143577edb4921b71ede051dac62ca3c16084e918bf7b40c96ae10eb33" +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "seq-macro" version = "0.3.6" @@ -4502,6 +4852,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "sse-stream" version = "0.2.1" @@ -5003,6 +5362,18 @@ dependencies = [ "termcolor", ] +[[package]] +name = "ttf-parser" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typedmap" version = "0.6.0" @@ -5269,12 +5640,44 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "weezl" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009936b22a61d342859b5f0ea64681cbb35a358ab548e2a44a8cf0dac2d980b8" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -5284,6 +5687,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.62.2" @@ -5559,6 +5968,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wio" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5" +dependencies = [ + "winapi", +] + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -5583,6 +6001,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yeslogic-fontconfig-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503a066b4c037c440169d995b869046827dbc71263f6e8f3be6d77d4f3229dbd" +dependencies = [ + "dlib", + "once_cell", + "pkg-config", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 9d4aefa6a..ec5ca2ca5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ aide = { version = "0.16.0-alpha.1", features = ["axum-json", "axum-query"] } axum = "0.8.6" bitcoin = { version = "0.32.7", features = ["serde"] } bitcoincore-rpc = "0.19.0" +brk_bencher = { version = "0.0.111", path = "crates/brk_bencher" } brk_binder = { version = "0.0.111", path = "crates/brk_binder" } brk_bundler = { version = "0.0.111", path = "crates/brk_bundler" } brk_cli = { version = "0.0.111", path = "crates/brk_cli" } @@ -60,12 +61,12 @@ brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_d byteview = "=0.6.1" # byteview = "~0.8.0" derive_deref = "1.1.1" -fjall2 = { version = "2.11.5", package = "brk_fjall" } -# fjall2 = { path = "../fjall2", package = "brk_fjall" } +# fjall2 = { version = "2.11.5", package = "brk_fjall" } +fjall2 = { path = "../fjall2", package = "brk_fjall" } # fjall2 = { version = "2.11.2", package = "fjall" } -fjall3 = { version = "=3.0.0-pre.4", package = "fjall" } -# fjall3 = { path = "../fjall3", package = "brk_fjall" } -# fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "bb15057500dce3115d7644d268b9deeaa895b431", package = "fjall" } +# fjall3 = { version = "=3.0.0-pre.5", package = "fjall" } +fjall3 = { path = "../fjall3", package = "fjall" } +# fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "f0bf96c2017b3543eb176012b8eff69c639dff1d", package = "fjall" } jiff = "0.2.16" log = "0.4.28" minreq = { version = "2.14.1", features = ["https", "serde_json"] } diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index 399affe6e..dc01f16c3 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -12,6 +12,7 @@ build = "build.rs" [features] default = ["cli"] full = [ + "bencher", "binder", "bundler", "cli", @@ -32,6 +33,7 @@ full = [ "traversable", "types", ] +bencher = ["brk_bencher"] binder = ["brk_binder"] bundler = ["brk_bundler"] cli = ["brk_cli"] @@ -53,6 +55,7 @@ traversable = ["brk_traversable"] types = ["brk_types"] [dependencies] +brk_bencher = { workspace = true, optional = true } brk_binder = { workspace = true, optional = true } brk_bundler = { workspace = true, optional = true } brk_cli = { workspace = true, optional = true } diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs index e0a157e26..137c49e1d 100644 --- a/crates/brk/src/lib.rs +++ b/crates/brk/src/lib.rs @@ -1,5 +1,9 @@ #![doc = include_str!("../README.md")] +#[cfg(feature = "bencher")] +#[doc(inline)] +pub use brk_bencher as bencher; + #[cfg(feature = "binder")] #[doc(inline)] pub use brk_binder as binder; diff --git a/crates/brk_bencher/Cargo.toml b/crates/brk_bencher/Cargo.toml new file mode 100644 index 000000000..9515c2d57 --- /dev/null +++ b/crates/brk_bencher/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "brk_bencher" +description = "A simple benchmarker for testing other crates." +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true +build = "build.rs" + +[dependencies] +brk_error = { workspace = true } diff --git a/crates/brk_bencher/README.md b/crates/brk_bencher/README.md new file mode 100644 index 000000000..c0863ffbc --- /dev/null +++ b/crates/brk_bencher/README.md @@ -0,0 +1 @@ +# brk_bencher diff --git a/crates/brk_bencher/build.rs b/crates/brk_bencher/build.rs new file mode 100644 index 000000000..a4055a31e --- /dev/null +++ b/crates/brk_bencher/build.rs @@ -0,0 +1,8 @@ +fn main() { + let profile = std::env::var("PROFILE").unwrap_or_default(); + + if profile == "release" { + println!("cargo:rustc-flag=-C"); + println!("cargo:rustc-flag=target-cpu=native"); + } +} diff --git a/crates/brk_bencher/src/lib.rs b/crates/brk_bencher/src/lib.rs new file mode 100644 index 000000000..cd634ed7b --- /dev/null +++ b/crates/brk_bencher/src/lib.rs @@ -0,0 +1,233 @@ +use std::{ + fs, + io::Write, + path::{Path, PathBuf}, + process::Command, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + thread::{self, JoinHandle}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use brk_error::Result; + +pub struct Bencher { + bench_dir: PathBuf, + stop_flag: Arc, + monitor_thread: Option>>, +} + +impl Bencher { + /// Create a new bencher for the given crate name + /// Creates directory structure: workspace_root/benches/{crate_name}/{timestamp}/ + pub fn new(crate_name: &str, workspace_root: &Path) -> Result { + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let bench_dir = workspace_root + .join("benches") + .join(crate_name) + .join(timestamp.to_string()); + + fs::create_dir_all(&bench_dir)?; + + Ok(Self { + bench_dir, + stop_flag: Arc::new(AtomicBool::new(false)), + monitor_thread: None, + }) + } + + /// Create a bencher using CARGO_MANIFEST_DIR to find workspace root + pub fn from_cargo_env() -> Result { + let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("Failed to find workspace root")?; + let crate_name = env!("CARGO_PKG_NAME"); + Self::new(crate_name, workspace_root) + } + + /// Start monitoring disk usage and memory footprint + pub fn start(&mut self) -> Result<()> { + if self.monitor_thread.is_some() { + return Err("Bencher already started".into()); + } + + let stop_flag = self.stop_flag.clone(); + let bench_dir = self.bench_dir.clone(); + + let handle = thread::spawn(move || monitor_resources(&bench_dir, stop_flag)); + + self.monitor_thread = Some(handle); + Ok(()) + } + + /// Stop monitoring and wait for the thread to finish + pub fn stop(mut self) -> Result<()> { + self.stop_flag.store(true, Ordering::Relaxed); + + if let Some(handle) = self.monitor_thread.take() { + handle.join().map_err(|_| "Monitor thread panicked")??; + } + + Ok(()) + } + + /// Get the benchmark output directory + pub fn bench_dir(&self) -> &Path { + &self.bench_dir + } +} + +fn parse_size_to_mb(value_str: &str, unit: &str) -> Option { + let value: f64 = value_str.parse().ok()?; + match unit { + "MB" | "M" => Some(value), + "GB" | "G" => Some(value * 1024.0), + "KB" | "K" => Some(value / 1024.0), + "B" => Some(value / 1024.0 / 1024.0), + _ => None, + } +} + +fn parse_du_output(size_str: &str) -> Option { + // Parse outputs like "524M", "287G", "4.0K" + let size_str = size_str.trim(); + + if let Some(unit_pos) = size_str.find(|c: char| c.is_alphabetic()) { + let (value_part, unit_part) = size_str.split_at(unit_pos); + parse_size_to_mb(value_part, unit_part) + } else { + // No unit means bytes + let value: f64 = size_str.parse().ok()?; + Some(value / 1024.0 / 1024.0) + } +} + +fn parse_footprint_output(output: &str) -> Option<(f64, f64)> { + let mut phys_footprint = None; + let mut phys_footprint_peak = None; + + for line in output.lines() { + if line.contains("phys_footprint:") && !line.contains("peak") { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 3 { + phys_footprint = parse_size_to_mb(parts[1], parts[2]); + } + } else if line.contains("phys_footprint_peak:") { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 3 { + phys_footprint_peak = parse_size_to_mb(parts[1], parts[2]); + } + } + } + + match (phys_footprint, phys_footprint_peak) { + (Some(f), Some(p)) => Some((f, p)), + _ => None, + } +} + +#[cfg(target_os = "linux")] +fn get_memory_usage_linux(pid: u32) -> Result<(f64, f64)> { + // Read /proc/[pid]/status for memory information + let status_path = format!("/proc/{}/status", pid); + let status_content = fs::read_to_string(status_path)?; + + let mut vm_rss = None; + let mut vm_hwm = None; + + for line in status_content.lines() { + if line.starts_with("VmRSS:") { + // Current RSS in kB + if let Some(value_str) = line.split_whitespace().nth(1) + && let Ok(kb) = value_str.parse::() + { + vm_rss = Some(kb / 1024.0); // Convert kB to MB + } + } else if line.starts_with("VmHWM:") { + // Peak RSS (High Water Mark) in kB + if let Some(value_str) = line.split_whitespace().nth(1) + && let Ok(kb) = value_str.parse::() + { + vm_hwm = Some(kb / 1024.0); // Convert kB to MB + } + } + } + + match (vm_rss, vm_hwm) { + (Some(rss), Some(hwm)) => Ok((rss, hwm)), + _ => Err("Failed to parse memory info from /proc/[pid]/status".into()), + } +} + +#[cfg(target_os = "macos")] +fn get_memory_usage_macos(pid: u32) -> Result<(f64, f64)> { + let output = Command::new("footprint") + .args(["-p", &pid.to_string()]) + .output()?; + + let stdout = String::from_utf8(output.stdout).unwrap(); + parse_footprint_output(&stdout).ok_or_else(|| "Failed to parse footprint output".into()) +} + +fn get_memory_usage(pid: u32) -> Result<(f64, f64)> { + #[cfg(target_os = "macos")] + { + get_memory_usage_macos(pid) + } + + #[cfg(target_os = "linux")] + { + get_memory_usage_linux(pid) + } + + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + Err("Unsupported platform for memory monitoring".into()) + } +} + +fn monitor_resources(bench_dir: &Path, stop_flag: Arc) -> Result<()> { + let disk_file = bench_dir.join("disk_usage.csv"); + let memory_file = bench_dir.join("memory_footprint.csv"); + + let mut disk_writer = fs::File::create(disk_file)?; + let mut memory_writer = fs::File::create(memory_file)?; + + writeln!(disk_writer, "timestamp_ms,disk_usage_mb")?; + writeln!( + memory_writer, + "timestamp_ms,phys_footprint_mb,phys_footprint_peak_mb" + )?; + + let pid = std::process::id(); + let start = Instant::now(); + + while !stop_flag.load(Ordering::Relaxed) { + let elapsed_ms = start.elapsed().as_millis(); + + // Get disk usage + if let Ok(output) = Command::new("du") + .args(["-sh", bench_dir.to_str().unwrap()]) + .output() + && let Ok(stdout) = String::from_utf8(output.stdout) + && let Some(size_str) = stdout.split_whitespace().next() + && let Some(size_mb) = parse_du_output(size_str) + { + writeln!(disk_writer, "{},{}", elapsed_ms, size_mb)?; + disk_writer.flush()?; + } + + // Get memory footprint (cross-platform) + if let Ok((footprint, peak)) = get_memory_usage(pid) { + writeln!(memory_writer, "{},{},{}", elapsed_ms, footprint, peak)?; + memory_writer.flush()?; + } + + thread::sleep(Duration::from_secs(1)); + } + + Ok(()) +} diff --git a/crates/brk_bencher_visualizer/Cargo.toml b/crates/brk_bencher_visualizer/Cargo.toml new file mode 100644 index 000000000..9cf3a18fb --- /dev/null +++ b/crates/brk_bencher_visualizer/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "brk_bencher_visualizer" +description = "A generator of charts for brk_bencher" +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true +build = "build.rs" + +[dependencies] +plotters = "0.3.7" diff --git a/crates/brk_bencher_visualizer/README.md b/crates/brk_bencher_visualizer/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/crates/brk_bencher_visualizer/build.rs b/crates/brk_bencher_visualizer/build.rs new file mode 100644 index 000000000..a4055a31e --- /dev/null +++ b/crates/brk_bencher_visualizer/build.rs @@ -0,0 +1,8 @@ +fn main() { + let profile = std::env::var("PROFILE").unwrap_or_default(); + + if profile == "release" { + println!("cargo:rustc-flag=-C"); + println!("cargo:rustc-flag=target-cpu=native"); + } +} diff --git a/crates/brk_bencher_visualizer/src/lib.rs b/crates/brk_bencher_visualizer/src/lib.rs new file mode 100644 index 000000000..56dadde14 --- /dev/null +++ b/crates/brk_bencher_visualizer/src/lib.rs @@ -0,0 +1,317 @@ +use plotters::prelude::*; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +type Result = std::result::Result>; + +#[derive(Debug, Clone)] +struct DataPoint { + timestamp_ms: u64, + value: f64, +} + +#[derive(Debug)] +struct BenchmarkRun { + run_id: String, + data: Vec, +} + +pub struct Visualizer { + workspace_root: PathBuf, +} + +impl Visualizer { + pub fn new(workspace_root: impl AsRef) -> Self { + Self { + workspace_root: workspace_root.as_ref().to_path_buf(), + } + } + + pub fn from_cargo_env() -> Result { + let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("Failed to find workspace root")? + .to_path_buf(); + Ok(Self { workspace_root }) + } + + /// Generate all charts for all crates in the benches directory + pub fn generate_all_charts(&self) -> Result<()> { + let benches_dir = self.workspace_root.join("benches"); + + if !benches_dir.exists() { + return Err("Benches directory does not exist".into()); + } + + // Iterate through each crate directory + for entry in fs::read_dir(&benches_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + let crate_name = path + .file_name() + .and_then(|n| n.to_str()) + .ok_or("Invalid crate name")?; + + println!("Generating charts for crate: {}", crate_name); + self.generate_crate_charts(&path, crate_name)?; + } + } + + Ok(()) + } + + /// Generate charts for a specific crate + fn generate_crate_charts(&self, crate_path: &Path, crate_name: &str) -> Result<()> { + // Read all benchmark runs for this crate + let disk_runs = self.read_benchmark_runs(crate_path, "disk_usage.csv")?; + let memory_runs = self.read_benchmark_runs(crate_path, "memory_footprint.csv")?; + + if !disk_runs.is_empty() { + self.generate_disk_chart(crate_path, crate_name, &disk_runs)?; + } + + if !memory_runs.is_empty() { + self.generate_memory_chart(crate_path, crate_name, &memory_runs)?; + } + + Ok(()) + } + + /// Read all benchmark runs from subdirectories + fn read_benchmark_runs(&self, crate_path: &Path, filename: &str) -> Result> { + let mut runs = Vec::new(); + + for entry in fs::read_dir(crate_path)? { + let entry = entry?; + let run_path = entry.path(); + + if run_path.is_dir() { + let run_id = run_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or("Invalid run ID")? + .to_string(); + + let csv_path = run_path.join(filename); + + if csv_path.exists() + && let Ok(data) = self.read_csv(&csv_path) + { + runs.push(BenchmarkRun { run_id, data }); + } + } + } + + Ok(runs) + } + + /// Read a CSV file and parse data points + fn read_csv(&self, path: &Path) -> Result> { + let content = fs::read_to_string(path)?; + let mut data = Vec::new(); + + for (i, line) in content.lines().enumerate() { + if i == 0 { + continue; + } // Skip header + + let parts: Vec<&str> = line.split(',').collect(); + if parts.len() >= 2 + && let (Ok(timestamp_ms), Ok(value)) = + (parts[0].parse::(), parts[1].parse::()) + { + data.push(DataPoint { + timestamp_ms, + value, + }); + } + } + + Ok(data) + } + + /// Generate disk usage chart + fn generate_disk_chart( + &self, + crate_path: &Path, + crate_name: &str, + runs: &[BenchmarkRun], + ) -> Result<()> { + let output_path = crate_path.join("disk_usage_chart.png"); + + let root = BitMapBackend::new(&output_path, (1200, 800)).into_drawing_area(); + root.fill(&WHITE)?; + + let max_time = runs + .iter() + .flat_map(|r| r.data.iter().map(|d| d.timestamp_ms)) + .max() + .unwrap_or(1000); + + let max_value = runs + .iter() + .flat_map(|r| r.data.iter().map(|d| d.value)) + .fold(0.0_f64, f64::max); + + let mut chart = ChartBuilder::on(&root) + .caption( + format!("{} - Disk Usage", crate_name), + ("sans-serif", 40).into_font(), + ) + .margin(10) + .x_label_area_size(40) + .y_label_area_size(60) + .build_cartesian_2d(0u64..max_time, 0.0..max_value * 1.1)?; + + chart + .configure_mesh() + .x_desc("Time (ms)") + .y_desc("Disk Usage (MB)") + .draw()?; + + let colors = [&RED, &BLUE, &GREEN, &CYAN, &MAGENTA, &YELLOW]; + + for (idx, run) in runs.iter().enumerate() { + let color = colors[idx % colors.len()]; + + chart + .draw_series(LineSeries::new( + run.data.iter().map(|d| (d.timestamp_ms, d.value)), + color, + ))? + .label(&run.run_id) + .legend(move |(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], color)); + } + + chart + .configure_series_labels() + .background_style(WHITE.mix(0.8)) + .border_style(BLACK) + .draw()?; + + root.present()?; + println!("Generated: {}", output_path.display()); + Ok(()) + } + + /// Generate memory footprint chart + fn generate_memory_chart( + &self, + crate_path: &Path, + crate_name: &str, + runs: &[BenchmarkRun], + ) -> Result<()> { + let output_path = crate_path.join("memory_footprint_chart.png"); + + let root = BitMapBackend::new(&output_path, (1200, 800)).into_drawing_area(); + root.fill(&WHITE)?; + + // Read memory CSV files which have 3 columns: timestamp, footprint, peak + let mut enhanced_runs = Vec::new(); + + for run in runs { + // Re-read the CSV to get both footprint and peak values + let csv_path = crate_path.join(&run.run_id).join("memory_footprint.csv"); + if let Ok(content) = fs::read_to_string(&csv_path) { + let mut footprint_data = Vec::new(); + let mut peak_data = Vec::new(); + + for (i, line) in content.lines().enumerate() { + if i == 0 { + continue; + } // Skip header + + let parts: Vec<&str> = line.split(',').collect(); + if parts.len() >= 3 + && let (Ok(timestamp_ms), Ok(footprint), Ok(peak)) = ( + parts[0].parse::(), + parts[1].parse::(), + parts[2].parse::(), + ) + { + footprint_data.push(DataPoint { + timestamp_ms, + value: footprint, + }); + peak_data.push(DataPoint { + timestamp_ms, + value: peak, + }); + } + } + + enhanced_runs.push((run.run_id.clone(), footprint_data, peak_data)); + } + } + + let max_time = enhanced_runs + .iter() + .flat_map(|(_, f, p)| f.iter().chain(p.iter()).map(|d| d.timestamp_ms)) + .max() + .unwrap_or(1000); + + let max_value = enhanced_runs + .iter() + .flat_map(|(_, f, p)| f.iter().chain(p.iter()).map(|d| d.value)) + .fold(0.0_f64, f64::max); + + let mut chart = ChartBuilder::on(&root) + .caption( + format!("{} - Memory Footprint", crate_name), + ("sans-serif", 40).into_font(), + ) + .margin(10) + .x_label_area_size(40) + .y_label_area_size(60) + .build_cartesian_2d(0u64..max_time, 0.0..max_value * 1.1)?; + + chart + .configure_mesh() + .x_desc("Time (ms)") + .y_desc("Memory (MB)") + .draw()?; + + let colors = [&RED, &BLUE, &GREEN, &CYAN, &MAGENTA, &YELLOW]; + + for (idx, (run_id, footprint_data, peak_data)) in enhanced_runs.iter().enumerate() { + let color = colors[idx % colors.len()]; + + // Draw footprint line (solid) + chart + .draw_series(LineSeries::new( + footprint_data.iter().map(|d| (d.timestamp_ms, d.value)), + color, + ))? + .label(format!("{} (current)", run_id)) + .legend(move |(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], color)); + + // Draw peak line (dashed) + chart + .draw_series(LineSeries::new( + peak_data.iter().map(|d| (d.timestamp_ms, d.value)), + color.stroke_width(2).filled(), + ))? + .label(format!("{} (peak)", run_id)) + .legend(move |(x, y)| { + PathElement::new( + vec![(x, y), (x + 10, y), (x + 20, y)], + color.stroke_width(2), + ) + }); + } + + chart + .configure_series_labels() + .background_style(WHITE.mix(0.8)) + .border_style(BLACK) + .draw()?; + + root.present()?; + println!("Generated: {}", output_path.display()); + Ok(()) + } +} diff --git a/crates/brk_bencher_visualizer/src/main.rs b/crates/brk_bencher_visualizer/src/main.rs new file mode 100644 index 000000000..e2645fadc --- /dev/null +++ b/crates/brk_bencher_visualizer/src/main.rs @@ -0,0 +1,6 @@ +use brk_bencher_visualizer::Visualizer; + +fn main() { + let v = Visualizer::from_cargo_env().unwrap(); + v.generate_all_charts().unwrap(); +} diff --git a/crates/brk_computer/src/blks.rs b/crates/brk_computer/src/blks.rs index 8e235f995..5ca876eca 100644 --- a/crates/brk_computer/src/blks.rs +++ b/crates/brk_computer/src/blks.rs @@ -43,7 +43,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/chain.rs b/crates/brk_computer/src/chain.rs index 6122d52d7..d67250022 100644 --- a/crates/brk_computer/src/chain.rs +++ b/crates/brk_computer/src/chain.rs @@ -1097,7 +1097,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index aaa73091d..c4d4f5368 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -279,7 +279,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index c4175a674..a9b67c417 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -162,7 +162,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index 308fc175f..13eababa9 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -44,7 +44,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs index c3a5bb354..4076b25de 100644 --- a/crates/brk_computer/src/grouped/builder_eager.rs +++ b/crates/brk_computer/src/grouped/builder_eager.rs @@ -534,7 +534,7 @@ where pub fn starting_index(&self, max_from: I) -> I { max_from.min(I::from( - self.iter_any_collectable().map(|v| v.len()).min().unwrap(), + self.iter_any_writable().map(|v| v.len()).min().unwrap(), )) } diff --git a/crates/brk_computer/src/grouped/builder_lazy.rs b/crates/brk_computer/src/grouped/builder_lazy.rs index 3ebf9ec8a..804bc1d45 100644 --- a/crates/brk_computer/src/grouped/builder_lazy.rs +++ b/crates/brk_computer/src/grouped/builder_lazy.rs @@ -209,7 +209,7 @@ where pub fn starting_index(&self, max_from: I) -> I { max_from.min(I::from( - self.iter_any_collectable().map(|v| v.len()).min().unwrap(), + self.iter_any_writable().map(|v| v.len()).min().unwrap(), )) } diff --git a/crates/brk_computer/src/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs index dc292432e..893b6b681 100644 --- a/crates/brk_computer/src/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -4,7 +4,7 @@ use brk_traversable::Traversable; use brk_types::{ DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec}; +use vecdb::{AnyWritableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec}; use crate::{Indexes, grouped::LazyVecsBuilder, indexes}; @@ -178,17 +178,17 @@ where .unwrap() } - fn iter_any_collectable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.dateindex_extra.iter_any_collectable()); - regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable())); + fn iter_any_writable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.dateindex_extra.iter_any_writable()); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable())); if let Some(ref x) = self.dateindex { - regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); } regular_iter } diff --git a/crates/brk_computer/src/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs index 3be396797..abadc5691 100644 --- a/crates/brk_computer/src/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -5,7 +5,7 @@ use brk_types::{ DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec}; +use vecdb::{AnyWritableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec}; use crate::{ Indexes, @@ -239,19 +239,19 @@ where .unwrap() } - fn iter_any_collectable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.height_extra.iter_any_collectable()); - regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable())); + fn iter_any_writable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height_extra.iter_any_writable()); + regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable())); if let Some(ref x) = self.height { - regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); } regular_iter } diff --git a/crates/brk_computer/src/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs index cd43640bf..04108e417 100644 --- a/crates/brk_computer/src/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -2,7 +2,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{DifficultyEpoch, Height, Version}; -use vecdb::{AnyCollectableVec, Database, EagerVec, Exit}; +use vecdb::{AnyWritableVec, Database, EagerVec, Exit}; use crate::{Indexes, indexes}; @@ -110,11 +110,11 @@ where .merge_branches() .unwrap() } - fn iter_any_collectable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.height.iter_any_collectable()); - regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable())); + fn iter_any_writable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height.iter_any_writable()); + regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable())); regular_iter } } diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index 62740dfb2..d61b5f7b9 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -6,7 +6,7 @@ use brk_types::{ Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex, }; use vecdb::{ - AnyCollectableVec, AnyVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, + AnyVec, AnyWritableVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, IterableCloneableVec, StoredIndex, TypedVecIterator, }; @@ -521,19 +521,19 @@ where .unwrap() } - fn iter_any_collectable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.height.iter_any_collectable()); - regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable())); - regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable())); + fn iter_any_writable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height.iter_any_writable()); + regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable())); if let Some(ref x) = self.txindex { - regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); } regular_iter } diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index eba5a1f54..3e58f5a99 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -492,7 +492,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/market.rs b/crates/brk_computer/src/market.rs index cde2d1d75..67e55a2e0 100644 --- a/crates/brk_computer/src/market.rs +++ b/crates/brk_computer/src/market.rs @@ -1517,7 +1517,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/pools/mod.rs b/crates/brk_computer/src/pools/mod.rs index 06cf11cd9..b0a4f85b4 100644 --- a/crates/brk_computer/src/pools/mod.rs +++ b/crates/brk_computer/src/pools/mod.rs @@ -62,7 +62,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/price.rs b/crates/brk_computer/src/price.rs index b6e9dd6ea..09ab7b202 100644 --- a/crates/brk_computer/src/price.rs +++ b/crates/brk_computer/src/price.rs @@ -327,7 +327,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 2ef31cfea..8ec6b686a 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -482,7 +482,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_grouper/src/address.rs b/crates/brk_grouper/src/address.rs index d62236414..1a908d5b3 100644 --- a/crates/brk_grouper/src/address.rs +++ b/crates/brk_grouper/src/address.rs @@ -1,6 +1,6 @@ use brk_traversable::Traversable; use rayon::prelude::*; -use vecdb::AnyCollectableVec; +use vecdb::AnyWritableVec; use crate::Filtered; @@ -78,14 +78,14 @@ where ) } - fn iter_any_collectable(&self) -> impl Iterator { + fn iter_any_writable(&self) -> impl Iterator { [ - Box::new(self.ge_amount.iter_any_collectable()) - as Box>, - Box::new(self.amount_range.iter_any_collectable()) - as Box>, - Box::new(self.lt_amount.iter_any_collectable()) - as Box>, + Box::new(self.ge_amount.iter_any_writable()) + as Box>, + Box::new(self.amount_range.iter_any_writable()) + as Box>, + Box::new(self.lt_amount.iter_any_writable()) + as Box>, ] .into_iter() .flatten() diff --git a/crates/brk_grouper/src/by_address_type.rs b/crates/brk_grouper/src/by_address_type.rs index adb84c1dc..1b6a09e95 100644 --- a/crates/brk_grouper/src/by_address_type.rs +++ b/crates/brk_grouper/src/by_address_type.rs @@ -4,7 +4,7 @@ use brk_error::Result; use brk_traversable::{Traversable, TreeNode}; use brk_types::OutputType; use rayon::prelude::*; -use vecdb::AnyCollectableVec; +use vecdb::AnyWritableVec; use super::{Filter, Filtered}; @@ -308,16 +308,16 @@ impl Traversable for ByAddressType { ) } - fn iter_any_collectable(&self) -> impl Iterator { - let mut iter: Box> = - Box::new(self.p2pk65.iter_any_collectable()); - iter = Box::new(iter.chain(self.p2pk33.iter_any_collectable())); - iter = Box::new(iter.chain(self.p2pkh.iter_any_collectable())); - iter = Box::new(iter.chain(self.p2sh.iter_any_collectable())); - iter = Box::new(iter.chain(self.p2wpkh.iter_any_collectable())); - iter = Box::new(iter.chain(self.p2wsh.iter_any_collectable())); - iter = Box::new(iter.chain(self.p2tr.iter_any_collectable())); - iter = Box::new(iter.chain(self.p2a.iter_any_collectable())); + fn iter_any_writable(&self) -> impl Iterator { + let mut iter: Box> = + Box::new(self.p2pk65.iter_any_writable()); + iter = Box::new(iter.chain(self.p2pk33.iter_any_writable())); + iter = Box::new(iter.chain(self.p2pkh.iter_any_writable())); + iter = Box::new(iter.chain(self.p2sh.iter_any_writable())); + iter = Box::new(iter.chain(self.p2wpkh.iter_any_writable())); + iter = Box::new(iter.chain(self.p2wsh.iter_any_writable())); + iter = Box::new(iter.chain(self.p2tr.iter_any_writable())); + iter = Box::new(iter.chain(self.p2a.iter_any_writable())); iter } } diff --git a/crates/brk_grouper/src/filter.rs b/crates/brk_grouper/src/filter.rs index 055ed48be..890c6a1b4 100644 --- a/crates/brk_grouper/src/filter.rs +++ b/crates/brk_grouper/src/filter.rs @@ -2,7 +2,7 @@ use std::ops::Range; use brk_traversable::{Traversable, TreeNode}; use brk_types::{HalvingEpoch, OutputType}; -use vecdb::AnyCollectableVec; +use vecdb::AnyWritableVec; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Filter { @@ -82,7 +82,7 @@ impl Traversable for Filtered { self.1.to_tree_node() } - fn iter_any_collectable(&self) -> impl Iterator { - self.1.iter_any_collectable() + fn iter_any_writable(&self) -> impl Iterator { + self.1.iter_any_writable() } } diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index b2a17ebcb..2e09e8a4f 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -11,6 +11,7 @@ build = "build.rs" [dependencies] bitcoin = { workspace = true } +brk_bencher = { workspace = true } brk_error = { workspace = true } brk_grouper = { workspace = true } brk_iterator = { workspace = true } @@ -26,4 +27,3 @@ log = { workspace = true } rayon = { workspace = true } rustc-hash = { workspace = true } vecdb = { workspace = true } -rand = "0.9.2" diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index e923ae918..47dbacb63 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -10,6 +10,7 @@ use brk_indexer::Indexer; use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; +use log::debug; use vecdb::Exit; fn main() -> Result<()> { @@ -28,12 +29,15 @@ fn main() -> Result<()> { )?; let reader = Reader::new(bitcoin_dir.join("blocks"), &client); + debug!("Reader created."); let blocks = Blocks::new(&client, &reader); + debug!("Blocks created."); fs::create_dir_all(&outputs_dir)?; let mut indexer = Indexer::forced_import(&outputs_dir)?; + debug!("Indexer imported."); let exit = Exit::new(); exit.set_ctrlc_handler(); diff --git a/crates/brk_indexer/examples/indexer_bench.rs b/crates/brk_indexer/examples/indexer_bench.rs new file mode 100644 index 000000000..c112e6218 --- /dev/null +++ b/crates/brk_indexer/examples/indexer_bench.rs @@ -0,0 +1,48 @@ +use std::{env, fs, path::Path, time::Instant}; + +use brk_bencher::Bencher; +use brk_error::Result; +use brk_indexer::Indexer; +use brk_iterator::Blocks; +use brk_reader::Reader; +use brk_rpc::{Auth, Client}; +use vecdb::Exit; + +fn main() -> Result<()> { + brk_logger::init(Some(Path::new(".log")))?; + + let bitcoin_dir = Client::default_bitcoin_path(); + // let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); + + let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk"); + fs::create_dir_all(&outputs_dir)?; + // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); + + let client = Client::new( + Client::default_url(), + Auth::CookieFile(bitcoin_dir.join(".cookie")), + )?; + + let reader = Reader::new(bitcoin_dir.join("blocks"), &client); + + let blocks = Blocks::new(&client, &reader); + + fs::create_dir_all(&outputs_dir)?; + + let mut indexer = Indexer::forced_import(&outputs_dir)?; + + let exit = Exit::new(); + exit.set_ctrlc_handler(); + + let mut bencher = Bencher::from_cargo_env()?; + bencher.start()?; + + let i = Instant::now(); + indexer.checked_index(&blocks, &client, &exit)?; + dbg!(i.elapsed()); + + // Stop and finalize + bencher.stop()?; + + Ok(()) +} diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index 56716848c..801ed843a 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -5,6 +5,7 @@ use brk_types::{ P2WPKHAddressIndex, P2WSHAddressIndex, TxInIndex, TxIndex, TxOutIndex, TypeIndex, UnknownOutputIndex, }; +use log::debug; use vecdb::{GenericStoredVec, IterableStoredVec, IterableVec, StoredIndex, StoredRaw}; use crate::{Stores, Vecs}; @@ -88,6 +89,8 @@ impl Indexes { impl From<(Height, &mut Vecs, &Stores)> for Indexes { #[inline] fn from((min_height, vecs, stores): (Height, &mut Vecs, &Stores)) -> Self { + debug!("Creating indexes from vecs and stores..."); + // Height at which we want to start: min last saved + 1 or 0 let vecs_starting_height = vecs.starting_height(); let stores_starting_height = stores.starting_height(); diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index e07c27813..bcf26cc0f 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -12,7 +12,7 @@ use brk_types::{ OutPoint, OutputType, Sats, StoredBool, Timestamp, TxInIndex, TxIndex, TxOutIndex, Txid, TxidPrefix, TypeIndex, Unit, Version, Vin, Vout, }; -use log::{error, info}; +use log::{debug, error, info}; use rayon::prelude::*; use rustc_hash::{FxHashMap, FxHashSet}; use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, TypedVecIterator}; @@ -80,7 +80,11 @@ impl Indexer { exit: &Exit, check_collisions: bool, ) -> Result { + debug!("Starting indexing..."); + let last_blockhash = self.vecs.height_to_blockhash.iter()?.last(); + debug!("Last block hash found."); + let (starting_indexes, prev_hash) = if let Some(hash) = last_blockhash { let (height, hash) = client.get_closest_valid_height(hash)?; let starting_indexes = @@ -93,15 +97,19 @@ impl Indexer { } else { (Indexes::default(), None) }; + info!("Starting indexes set."); let lock = exit.lock(); self.stores .rollback_if_needed(&mut self.vecs, &starting_indexes)?; + debug!("Rollback stores done."); self.vecs.rollback_if_needed(&starting_indexes)?; + debug!("Rollback vecs done."); drop(lock); // Cloned because we want to return starting indexes for the computer let mut indexes = starting_indexes.clone(); + debug!("Indexes cloned."); let should_export = |height: Height, rem: bool| -> bool { height != 0 && (height % SNAPSHOT_BLOCK_RANGE == 0) != rem diff --git a/crates/brk_indexer/src/stores_v2.rs b/crates/brk_indexer/src/stores_v2.rs index 498c93401..b642befe7 100644 --- a/crates/brk_indexer/src/stores_v2.rs +++ b/crates/brk_indexer/src/stores_v2.rs @@ -20,12 +20,8 @@ use super::Vecs; pub struct Stores { pub keyspace: TransactionalKeyspace, - // pub addresshash_to_typeindex: Store, pub addresstype_to_addresshash_to_addressindex: ByAddressType>, - // pub addresstype_to_addressindex_and_txindex: Store, pub addresstype_to_addressindex_and_txindex: ByAddressType>, - // pub addresstype_to_addressindex_and_unspentoutpoint: - // Store, pub addresstype_to_addressindex_and_unspentoutpoint: ByAddressType>, pub blockhashprefix_to_height: Store, @@ -94,14 +90,6 @@ impl Stores { Mode::UniquePushOnly(Type::Sequential), Compression::Lz4, )?, - // addresshash_to_typeindex: Store::import( - // keyspace_ref, - // path, - // "a2t", - // version, - // Mode::UniquePushOnly(Type::Random), - // Compression::Lz4, - // )?, addresstype_to_addresshash_to_addressindex: ByAddressType::new_with_index( create_addresshash_to_addressindex_store, )?, @@ -121,25 +109,9 @@ impl Stores { Mode::UniquePushOnly(Type::Random), Compression::Lz4, )?, - // addresstype_to_addressindex_and_txindex: Store::import( - // keyspace_ref, - // path, - // "aat", - // version, - // Mode::VecLike, - // Compression::Lz4, - // )?, addresstype_to_addressindex_and_txindex: ByAddressType::new_with_index( create_addressindex_to_txindex_store, )?, - // addresstype_to_addressindex_and_unspentoutpoint: Store::import( - // keyspace_ref, - // path, - // "aau", - // version, - // Mode::VecLike, - // Compression::Lz4, - // )?, addresstype_to_addressindex_and_unspentoutpoint: ByAddressType::new_with_index( create_addressindex_to_unspentoutpoint_store, )?, @@ -158,13 +130,10 @@ impl Stores { } pub fn commit(&mut self, height: Height) -> Result<()> { - [ + let tuples = [ &mut self.blockhashprefix_to_height as &mut dyn AnyStore, &mut self.height_to_coinbase_tag, &mut self.txidprefix_to_txindex, - // &mut self.addresshash_to_typeindex - // &mut self.addresstype_to_addressindex_and_txindex, - // &mut self.addresstype_to_addressindex_and_unspentoutpoint, ] .into_par_iter() .chain( @@ -182,7 +151,15 @@ impl Stores { .par_iter_mut() .map(|s| s as &mut dyn AnyStore), ) // Changed from par_iter_mut() - .try_for_each(|store| store.commit(height))?; + .map(|store| { + let items = store.take_all_f2(); + store.export_meta_if_needed(height)?; + Ok((store.partition(), items)) + }) + .collect::>>()?; + + let batch = self.keyspace.inner().batch(); + batch.commit_partitions(tuples)?; self.keyspace .persist(PersistMode::SyncAll) @@ -194,9 +171,6 @@ impl Stores { &self.blockhashprefix_to_height as &dyn AnyStore, &self.height_to_coinbase_tag, &self.txidprefix_to_txindex, - // &self.addresshash_to_typeindex, - // &self.addresstype_to_addressindex_and_txindex, - // &self.addresstype_to_addressindex_and_unspentoutpoint, ] .into_iter() .chain( @@ -224,11 +198,6 @@ impl Stores { if self.blockhashprefix_to_height.is_empty()? && self.txidprefix_to_txindex.is_empty()? && self.height_to_coinbase_tag.is_empty()? - // && self.addresshash_to_typeindex.is_empty()? - // && self.addresstype_to_addressindex_and_txindex.is_empty()? - // && self - // .addresstype_to_addressindex_and_unspentoutpoint - // .is_empty()? && self .addresstype_to_addresshash_to_addressindex .iter() diff --git a/crates/brk_indexer/src/stores_v3.rs b/crates/brk_indexer/src/stores_v3.rs index a5fae1765..e927e03d1 100644 --- a/crates/brk_indexer/src/stores_v3.rs +++ b/crates/brk_indexer/src/stores_v3.rs @@ -1,11 +1,12 @@ use std::{fs, path::Path}; use brk_error::Result; +use brk_grouper::ByAddressType; use brk_store::{AnyStore, Kind3, Mode3, StoreFjallV3 as Store}; use brk_types::{ - AddressBytes, AddressHash, AddressTypeAddressIndexOutPoint, AddressTypeAddressIndexTxIndex, - BlockHashPrefix, Height, OutPoint, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, - Unit, Version, Vout, + AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height, + OutPoint, OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, + Vout, }; use fjall3::{Database, PersistMode}; use rayon::prelude::*; @@ -19,11 +20,15 @@ use super::Vecs; pub struct Stores { pub database: Database, - pub addresshash_to_typeindex: Store, + pub addresstype_to_addresshash_to_addressindex: ByAddressType>, + pub addresstype_to_addressindex_and_txindex: ByAddressType>, + pub addresstype_to_addressindex_and_unspentoutpoint: + ByAddressType>, pub blockhashprefix_to_height: Store, pub height_to_coinbase_tag: Store, pub txidprefix_to_txindex: Store, - pub addresstype_to_addressindex_and_txindex: Store, + // pub addresstype_to_addressindex_and_txindex: Store, + // pub addresshash_to_typeindex: Store, // pub addresstype_to_addressindex_and_unspentoutpoint: // Store, } @@ -45,6 +50,39 @@ impl Stores { let database_ref = &database; + let create_addresshash_to_addressindex_store = |index| { + Store::import( + database_ref, + path, + &format!("h2i{}", index), + version, + Mode3::PushOnly, + Kind3::Random, + ) + }; + + let create_addressindex_to_txindex_store = |index| { + Store::import( + database_ref, + path, + &format!("a2t{}", index), + version, + Mode3::PushOnly, + Kind3::Vec, + ) + }; + + let create_addressindex_to_unspentoutpoint_store = |index| { + Store::import( + database_ref, + path, + &format!("a2u{}", index), + version, + Mode3::Any, + Kind3::Vec, + ) + }; + Ok(Self { database: database.clone(), @@ -56,14 +94,23 @@ impl Stores { Mode3::PushOnly, Kind3::Sequential, )?, - addresshash_to_typeindex: Store::import( - database_ref, - path, - "addresshash_to_typeindex", - version, - Mode3::PushOnly, - Kind3::Random, + addresstype_to_addresshash_to_addressindex: ByAddressType::new_with_index( + create_addresshash_to_addressindex_store, )?, + addresstype_to_addressindex_and_txindex: ByAddressType::new_with_index( + create_addressindex_to_txindex_store, + )?, + addresstype_to_addressindex_and_unspentoutpoint: ByAddressType::new_with_index( + create_addressindex_to_unspentoutpoint_store, + )?, + // addresshash_to_typeindex: Store::import( + // database_ref, + // path, + // "addresshash_to_typeindex", + // version, + // Mode3::PushOnly, + // Kind3::Random, + // )?, blockhashprefix_to_height: Store::import( database_ref, path, @@ -80,14 +127,14 @@ impl Stores { Mode3::PushOnly, Kind3::Random, )?, - addresstype_to_addressindex_and_txindex: Store::import( - database_ref, - path, - "addresstype_to_addressindex_and_txindex", - version, - Mode3::PushOnly, - Kind3::Vec, - )?, + // addresstype_to_addressindex_and_txindex: Store::import( + // database_ref, + // path, + // "addresstype_to_addressindex_and_txindex", + // version, + // Mode3::PushOnly, + // Kind3::Vec, + // )?, // addresstype_to_addressindex_and_unspentoutpoint: Store::import( // database_ref, // path, @@ -112,14 +159,29 @@ impl Stores { pub fn commit(&mut self, height: Height) -> Result<()> { [ - &mut self.addresshash_to_typeindex as &mut dyn AnyStore, - &mut self.blockhashprefix_to_height, + &mut self.blockhashprefix_to_height as &mut dyn AnyStore, &mut self.height_to_coinbase_tag, &mut self.txidprefix_to_txindex, - &mut self.addresstype_to_addressindex_and_txindex, + // &mut self.addresshash_to_typeindex + // &mut self.addresstype_to_addressindex_and_txindex, // &mut self.addresstype_to_addressindex_and_unspentoutpoint, ] - .into_par_iter() // Changed from par_iter_mut() + .into_par_iter() + .chain( + self.addresstype_to_addresshash_to_addressindex + .par_iter_mut() + .map(|s| s as &mut dyn AnyStore), + ) + .chain( + self.addresstype_to_addressindex_and_txindex + .par_iter_mut() + .map(|s| s as &mut dyn AnyStore), + ) + .chain( + self.addresstype_to_addressindex_and_unspentoutpoint + .par_iter_mut() + .map(|s| s as &mut dyn AnyStore), + ) // Changed from par_iter_mut() .try_for_each(|store| store.commit(height))?; self.database @@ -129,14 +191,29 @@ impl Stores { fn iter_any_store(&self) -> impl Iterator { [ - &self.addresshash_to_typeindex as &dyn AnyStore, - &self.blockhashprefix_to_height, + &self.blockhashprefix_to_height as &dyn AnyStore, &self.height_to_coinbase_tag, &self.txidprefix_to_txindex, - &self.addresstype_to_addressindex_and_txindex, + // &self.addresshash_to_typeindex, + // &self.addresstype_to_addressindex_and_txindex, // &self.addresstype_to_addressindex_and_unspentoutpoint, ] .into_iter() + .chain( + self.addresstype_to_addresshash_to_addressindex + .iter() + .map(|s| s as &dyn AnyStore), + ) + .chain( + self.addresstype_to_addressindex_and_txindex + .iter() + .map(|s| s as &dyn AnyStore), + ) + .chain( + self.addresstype_to_addressindex_and_unspentoutpoint + .iter() + .map(|s| s as &dyn AnyStore), + ) } pub fn rollback_if_needed( @@ -144,14 +221,26 @@ impl Stores { vecs: &mut Vecs, starting_indexes: &Indexes, ) -> Result<()> { - if self.addresshash_to_typeindex.is_empty()? - && self.blockhashprefix_to_height.is_empty()? + if self.blockhashprefix_to_height.is_empty()? && self.txidprefix_to_txindex.is_empty()? && self.height_to_coinbase_tag.is_empty()? - && self.addresstype_to_addressindex_and_txindex.is_empty()? - // && self - // .addresstype_to_addressindex_and_unspentoutpoint - // .is_empty()? + // && self.addresshash_to_typeindex.is_empty()? + // && self.addresstype_to_addressindex_and_txindex.is_empty()? + // && self + // .addresstype_to_addressindex_and_unspentoutpoint + // .is_empty()? + && self + .addresstype_to_addresshash_to_addressindex + .iter() + .try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))? + && self + .addresstype_to_addressindex_and_txindex + .iter() + .try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))? + && self + .addresstype_to_addressindex_and_unspentoutpoint + .iter() + .try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))? { return Ok(()); } @@ -173,7 +262,7 @@ impl Stores { if let Ok(mut index) = vecs .height_to_first_p2pk65addressindex - .read(starting_indexes.height) + .read_once(starting_indexes.height) { let mut p2pk65addressindex_to_p2pk65bytes_iter = vecs.p2pk65addressindex_to_p2pk65bytes.iter()?; @@ -181,14 +270,16 @@ impl Stores { while let Some(typedbytes) = p2pk65addressindex_to_p2pk65bytes_iter.get(index) { let bytes = AddressBytes::from(typedbytes); let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2PK65) + .remove(hash); index.increment(); } } if let Ok(mut index) = vecs .height_to_first_p2pk33addressindex - .read(starting_indexes.height) + .read_once(starting_indexes.height) { let mut p2pk33addressindex_to_p2pk33bytes_iter = vecs.p2pk33addressindex_to_p2pk33bytes.iter()?; @@ -196,14 +287,16 @@ impl Stores { while let Some(typedbytes) = p2pk33addressindex_to_p2pk33bytes_iter.get(index) { let bytes = AddressBytes::from(typedbytes); let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2PK33) + .remove(hash); index.increment(); } } if let Ok(mut index) = vecs .height_to_first_p2pkhaddressindex - .read(starting_indexes.height) + .read_once(starting_indexes.height) { let mut p2pkhaddressindex_to_p2pkhbytes_iter = vecs.p2pkhaddressindex_to_p2pkhbytes.iter()?; @@ -211,14 +304,16 @@ impl Stores { while let Some(typedbytes) = p2pkhaddressindex_to_p2pkhbytes_iter.get(index) { let bytes = AddressBytes::from(typedbytes); let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2PKH) + .remove(hash); index.increment(); } } if let Ok(mut index) = vecs .height_to_first_p2shaddressindex - .read(starting_indexes.height) + .read_once(starting_indexes.height) { let mut p2shaddressindex_to_p2shbytes_iter = vecs.p2shaddressindex_to_p2shbytes.iter()?; @@ -226,29 +321,16 @@ impl Stores { while let Some(typedbytes) = p2shaddressindex_to_p2shbytes_iter.get(index) { let bytes = AddressBytes::from(typedbytes); let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); - index.increment(); - } - } - - if let Ok(mut index) = vecs - .height_to_first_p2traddressindex - .read(starting_indexes.height) - { - let mut p2traddressindex_to_p2trbytes_iter = - vecs.p2traddressindex_to_p2trbytes.iter()?; - - while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter.get(index) { - let bytes = AddressBytes::from(typedbytes); - let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2SH) + .remove(hash); index.increment(); } } if let Ok(mut index) = vecs .height_to_first_p2wpkhaddressindex - .read(starting_indexes.height) + .read_once(starting_indexes.height) { let mut p2wpkhaddressindex_to_p2wpkhbytes_iter = vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter()?; @@ -256,14 +338,16 @@ impl Stores { while let Some(typedbytes) = p2wpkhaddressindex_to_p2wpkhbytes_iter.get(index) { let bytes = AddressBytes::from(typedbytes); let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2WPKH) + .remove(hash); index.increment(); } } if let Ok(mut index) = vecs .height_to_first_p2wshaddressindex - .read(starting_indexes.height) + .read_once(starting_indexes.height) { let mut p2wshaddressindex_to_p2wshbytes_iter = vecs.p2wshaddressindex_to_p2wshbytes.iter()?; @@ -271,14 +355,33 @@ impl Stores { while let Some(typedbytes) = p2wshaddressindex_to_p2wshbytes_iter.get(index) { let bytes = AddressBytes::from(typedbytes); let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2WSH) + .remove(hash); + index.increment(); + } + } + + if let Ok(mut index) = vecs + .height_to_first_p2traddressindex + .read_once(starting_indexes.height) + { + let mut p2traddressindex_to_p2trbytes_iter = + vecs.p2traddressindex_to_p2trbytes.iter()?; + + while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter.get(index) { + let bytes = AddressBytes::from(typedbytes); + let hash = AddressHash::from(&bytes); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2TR) + .remove(hash); index.increment(); } } if let Ok(mut index) = vecs .height_to_first_p2aaddressindex - .read(starting_indexes.height) + .read_once(starting_indexes.height) { let mut p2aaddressindex_to_p2abytes_iter = vecs.p2aaddressindex_to_p2abytes.iter()?; @@ -286,7 +389,9 @@ impl Stores { while let Some(typedbytes) = p2aaddressindex_to_p2abytes_iter.get(index) { let bytes = AddressBytes::from(typedbytes); let hash = AddressHash::from(&bytes); - self.addresshash_to_typeindex.remove(hash); + self.addresstype_to_addresshash_to_addressindex + .get_mut_unwrap(OutputType::P2A) + .remove(hash); index.increment(); } } @@ -339,9 +444,9 @@ impl Stores { .for_each(|((txoutindex, addresstype), addressindex)| { let txindex = txoutindex_to_txindex_iter.get_at_unwrap(txoutindex); - self.addresstype_to_addressindex_and_txindex.remove( - AddressTypeAddressIndexTxIndex::from((addresstype, addressindex, txindex)), - ); + self.addresstype_to_addressindex_and_txindex + .get_mut_unwrap(addresstype) + .remove(AddressIndexTxIndex::from((addressindex, txindex))); let vout = Vout::from( txoutindex.to_usize() @@ -351,13 +456,9 @@ impl Stores { ); let outpoint = OutPoint::new(txindex, vout); - // self.addresstype_to_addressindex_and_unspentoutpoint.remove( - // AddressTypeAddressIndexOutPoint::from(( - // addresstype, - // addressindex, - // outpoint, - // )), - // ); + self.addresstype_to_addressindex_and_unspentoutpoint + .get_mut_unwrap(addresstype) + .remove(AddressIndexOutPoint::from((addressindex, outpoint))); }); // Add back outputs that were spent after the rollback point @@ -386,22 +487,13 @@ impl Stores { let addresstype = outputtype; let addressindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex); - self.addresstype_to_addressindex_and_txindex.remove( - AddressTypeAddressIndexTxIndex::from(( - addresstype, - addressindex, - txindex, - )), - ); + self.addresstype_to_addressindex_and_txindex + .get_mut_unwrap(addresstype) + .remove(AddressIndexTxIndex::from((addressindex, txindex))); - // self.addresstype_to_addressindex_and_unspentoutpoint.insert( - // AddressTypeAddressIndexOutPoint::from(( - // addresstype, - // addressindex, - // outpoint, - // )), - // Unit, - // ); + self.addresstype_to_addressindex_and_unspentoutpoint + .get_mut_unwrap(addresstype) + .insert(AddressIndexOutPoint::from((addressindex, outpoint)), Unit); } } }); diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index aba0cc2fa..e12809082 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -192,7 +192,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_collectable() + this.iter_any_writable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index d327b94b4..652af75ff 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -23,8 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> { }); Builder::from_env(Env::default().default_filter_or( - "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", - // "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", + // "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", + "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", )) .format(move |buf, record| { let date_time = Timestamp::now() diff --git a/crates/brk_query/src/async.rs b/crates/brk_query/src/async.rs index 1bcf67768..04fbbc4d2 100644 --- a/crates/brk_query/src/async.rs +++ b/crates/brk_query/src/async.rs @@ -53,14 +53,14 @@ impl AsyncQuery { // metric: &str, // index: Index, // // params: &Params, - // ) -> Result> { + // ) -> Result> { // let query = self.0.clone(); // spawn_blocking(move || query.search_metric_with_index(metric, index)).await? // } // pub async fn format( // &self, - // metrics: Vec<(String, &&dyn AnyCollectableVec)>, + // metrics: Vec<(String, &&dyn AnyWritableVec)>, // params: &ParamsOpt, // ) -> Result { // let query = self.0.clone(); diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index 697c1a7cd..2c32e76b0 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -11,7 +11,7 @@ use brk_types::{ Address, AddressStats, Format, Height, Index, IndexInfo, Limit, Metric, MetricCount, Transaction, TxidPath, }; -use vecdb::{AnyCollectableVec, AnyStoredVec}; +use vecdb::{AnyStoredVec, AnyWritableVec}; mod r#async; mod chain; @@ -77,7 +77,7 @@ impl Query { metric: &str, index: Index, // params: &Params, - ) -> Result> { + ) -> Result> { todo!(); // let all_metrics = &self.vecs.metrics; @@ -121,7 +121,7 @@ impl Query { } fn columns_to_csv( - columns: &[&&dyn AnyCollectableVec], + columns: &[&&dyn AnyWritableVec], from: Option, to: Option, ) -> Result { @@ -144,23 +144,31 @@ impl Query { } csv.push('\n'); - let mut iters: Vec<_> = columns + // Create one writer per column + let mut writers: Vec<_> = columns .iter() - .map(|col| col.iter_range_strings(from, to)) + .map(|col| col.create_writer(from, to)) .collect(); for _ in 0..num_rows { - for (index, iter) in iters.iter_mut().enumerate() { + for (index, writer) in writers.iter_mut().enumerate() { if index > 0 { csv.push(','); } - if let Some(field) = iter.next() { - if field.contains(',') { + + // Check if we need CSV escaping + let start_pos = csv.len(); + + if writer.write_next(&mut csv)? { + let end_pos = csv.len(); + + // If contains comma, rewrite with quotes + if csv[start_pos..end_pos].contains(',') { + let value = csv[start_pos..end_pos].to_string(); // Only allocate if needed + csv.truncate(start_pos); csv.push('"'); - csv.push_str(&field); + csv.push_str(&value); csv.push('"'); - } else { - csv.push_str(&field); } } } @@ -170,11 +178,7 @@ impl Query { Ok(csv) } - pub fn format( - &self, - metrics: Vec<&&dyn AnyCollectableVec>, - params: &ParamsOpt, - ) -> Result { + pub fn format(&self, metrics: Vec<&&dyn AnyWritableVec>, params: &ParamsOpt) -> Result { let from = params.from().map(|from| { metrics .iter() diff --git a/crates/brk_query/src/vecs.rs b/crates/brk_query/src/vecs.rs index a456d1d32..4908a26ac 100644 --- a/crates/brk_query/src/vecs.rs +++ b/crates/brk_query/src/vecs.rs @@ -6,7 +6,7 @@ use brk_traversable::{Traversable, TreeNode}; use brk_types::{Index, IndexInfo, Limit, Metric}; use derive_deref::{Deref, DerefMut}; use quickmatch::{QuickMatch, QuickMatchConfig}; -use vecdb::AnyCollectableVec; +use vecdb::AnyWritableVec; use crate::pagination::{PaginatedIndexParam, PaginatedMetrics, PaginationParam}; @@ -31,11 +31,11 @@ impl<'a> Vecs<'a> { indexer .vecs - .iter_any_collectable() + .iter_any_writable() .for_each(|vec| this.insert(vec)); computer - .iter_any_collectable() + .iter_any_writable() .for_each(|vec| this.insert(vec)); let mut ids = this @@ -108,7 +108,7 @@ impl<'a> Vecs<'a> { } // Not the most performant or type safe but only built once so that's okay - fn insert(&mut self, vec: &'a dyn AnyCollectableVec) { + fn insert(&mut self, vec: &'a dyn AnyWritableVec) { let name = vec.name(); let serialized_index = vec.index_type_to_string(); let index = Index::try_from(serialized_index) @@ -181,7 +181,7 @@ impl<'a> Vecs<'a> { } #[derive(Default, Deref, DerefMut)] -pub struct IndexToVec<'a>(BTreeMap); +pub struct IndexToVec<'a>(BTreeMap); #[derive(Default, Deref, DerefMut)] -pub struct MetricToVec<'a>(BTreeMap<&'a str, &'a dyn AnyCollectableVec>); +pub struct MetricToVec<'a>(BTreeMap<&'a str, &'a dyn AnyWritableVec>); diff --git a/crates/brk_rpc/src/lib.rs b/crates/brk_rpc/src/lib.rs index 283ba7c22..62008134a 100644 --- a/crates/brk_rpc/src/lib.rs +++ b/crates/brk_rpc/src/lib.rs @@ -17,7 +17,7 @@ pub use bitcoincore_rpc::Auth; mod inner; use inner::ClientInner; -use log::info; +use log::{debug, info}; /// /// Bitcoin Core RPC Client @@ -67,6 +67,7 @@ impl Client { /// Returns the numbers of block in the longest chain. pub fn get_last_height(&self) -> Result { + debug!("Get last height..."); self.call(|c| c.get_block_count()) .map(Height::from) .map_err(Into::into) @@ -215,6 +216,8 @@ impl Client { } pub fn get_closest_valid_height(&self, hash: BlockHash) -> Result<(Height, BlockHash)> { + debug!("Get closest valid height..."); + match self.get_block_header_info(&hash) { Ok(block_info) => { if self.is_in_main_chain(&hash)? { diff --git a/crates/brk_store/src/any.rs b/crates/brk_store/src/any.rs index 46adefe73..f1946386a 100644 --- a/crates/brk_store/src/any.rs +++ b/crates/brk_store/src/any.rs @@ -1,11 +1,16 @@ use brk_error::Result; use brk_types::{Height, Version}; +use fjall2::{InnerItem, PartitionHandle}; pub trait AnyStore: Send + Sync { - fn commit(&mut self, height: Height) -> Result<()>; fn name(&self) -> &'static str; fn height(&self) -> Option; fn has(&self, height: Height) -> bool; fn needs(&self, height: Height) -> bool; fn version(&self) -> Version; + fn export_meta_if_needed(&mut self, height: Height) -> Result<()>; + fn partition(&self) -> &PartitionHandle; + fn take_all_f2(&mut self) -> Vec; + fn commit(&mut self) -> Result<()>; + // fn take_all_f3(&mut self) -> Box>; } diff --git a/crates/brk_store/src/fjall_v2/mod.rs b/crates/brk_store/src/fjall_v2/mod.rs index 13b5a7df5..9eceb5f64 100644 --- a/crates/brk_store/src/fjall_v2/mod.rs +++ b/crates/brk_store/src/fjall_v2/mod.rs @@ -169,18 +169,42 @@ where impl AnyStore for StoreFjallV2 where - K: Debug + Clone + From + Ord + Eq + Hash, - V: Debug + Clone + From, + K: Debug + Clone + From + Ord + Eq + Hash + 'static, + V: Debug + Clone + From + 'static, ByteView: From + From, Self: Send + Sync, { - fn commit(&mut self, height: Height) -> Result<()> { + fn partition(&self) -> &fjall2::PartitionHandle { + self.partition.inner() + } + + fn take_all_f2(&mut self) -> Vec { + let mut items = mem::take(&mut self.puts) + .into_iter() + .map(|(key, value)| Item::Value { key, value }) + .chain( + mem::take(&mut self.dels) + .into_iter() + .map(|key| Item::Tomb(key)), + ) + .collect::>(); + items.sort_unstable(); + items.into_iter().map(InnerItem::from).collect() + } + + // fn take_all_f3(&mut self) -> Box> { + // Box::new([].into_iter()) + // } + + fn export_meta_if_needed(&mut self, height: Height) -> Result<()> { if self.has(height) { return Ok(()); } - self.meta.export(height)?; + Ok(()) + } + fn commit(&mut self) -> Result<()> { if self.puts.is_empty() && self.dels.is_empty() { return Ok(()); } @@ -208,10 +232,10 @@ where .collect::>(); items.sort_unstable(); - self.keyspace.inner().batch().commit_partition( - self.partition.inner(), - items.into_iter().map(InnerItem::from).collect::>(), - )?; + // self.keyspace.inner().batch().commit_partition( + // self.partition.inner(), + // items.into_iter().map(InnerItem::from).collect::>(), + // )?; // } Ok(()) diff --git a/crates/brk_store/src/fjall_v3/mod.rs b/crates/brk_store/src/fjall_v3/mod.rs index b39422ca4..401b34aec 100644 --- a/crates/brk_store/src/fjall_v3/mod.rs +++ b/crates/brk_store/src/fjall_v3/mod.rs @@ -26,15 +26,13 @@ pub struct StoreFjallV3 { keyspace: Keyspace, puts: FxHashMap, dels: FxHashSet, - mode: Mode3, - kind: Kind3, } const MAJOR_FJALL_VERSION: Version = Version::new(3); pub fn open_fjall3_database(path: &Path) -> fjall3::Result { Database::builder(path.join("fjall")) - .cache_size(1024 * 1024 * 1024) + .cache_size(4 * 1024 * 1024 * 1024) .open() } @@ -73,8 +71,6 @@ where keyspace, puts: FxHashMap::default(), dels: FxHashSet::default(), - mode, - kind, }) } @@ -94,7 +90,9 @@ where FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(7.0)), ])); } else { - options = options.filter_policy(FilterPolicy::disabled()); + options = options + .max_memtable_size(8 * 1024 * 1024) + .filter_policy(FilterPolicy::disabled()); } if kind.is_sequential() { @@ -156,6 +154,14 @@ where } } + #[inline] + pub fn iter(&self) -> impl Iterator { + self.keyspace + .iter() + .map(|res| res.into_inner().unwrap()) + .map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v)))) + } + #[inline] fn has(&self, height: Height) -> bool { self.meta.has(height) @@ -174,49 +180,63 @@ where ByteView: From + From, Self: Send + Sync, { - fn commit(&mut self, height: Height) -> Result<()> { + fn take_all_f2(&mut self) -> Vec { + vec![] + } + + fn partition(&self) -> &fjall2::PartitionHandle { + panic!() + } + + // fn take_all_f3(&mut self) -> Box> { + // Box::new([].into_iter()) + // } + + fn export_meta_if_needed(&mut self, height: Height) -> Result<()> { if self.has(height) { return Ok(()); } - self.meta.export(height)?; + Ok(()) + } + fn commit(&mut self) -> Result<()> { if self.puts.is_empty() && self.dels.is_empty() { return Ok(()); } - if self.mode.is_push_only() { - if !self.dels.is_empty() { - unreachable!(); + // if self.mode.is_push_only() { + // if !self.dels.is_empty() { + // unreachable!(); + // } + // let mut puts = mem::take(&mut self.puts).into_iter().collect::>(); + // puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2)); + // // dbg!(&puts); + // self.keyspace.ingest( + // puts.into_iter() + // .map(|(k, v)| (ByteView::from(k), ByteView::from(v))), + // )?; + // } else { + let mut batch = self.database.batch(); + // let mut batch = self.database.inner().batch(); + let mut items = mem::take(&mut self.puts) + .into_iter() + .map(|(key, value)| Item::Value { key, value }) + .chain( + mem::take(&mut self.dels) + .into_iter() + .map(|key| Item::Tomb(key)), + ) + .collect::>(); + items.sort_unstable(); + items.into_iter().for_each(|item| match item { + Item::Value { key, value } => { + batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value)) } - let mut puts = mem::take(&mut self.puts).into_iter().collect::>(); - puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2)); - // dbg!(&puts); - self.keyspace.ingest( - puts.into_iter() - .map(|(k, v)| (ByteView::from(k), ByteView::from(v))), - )?; - } else { - let mut batch = self.database.batch(); - // let mut batch = self.database.inner().batch(); - let mut items = mem::take(&mut self.puts) - .into_iter() - .map(|(key, value)| Item::Value { key, value }) - .chain( - mem::take(&mut self.dels) - .into_iter() - .map(|key| Item::Tomb(key)), - ) - .collect::>(); - items.sort_unstable(); - items.into_iter().for_each(|item| match item { - Item::Value { key, value } => { - batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value)) - } - Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)), - }); - batch.commit()?; - } + Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)), + }); + batch.commit()?; + // } // batch.ingest( // items diff --git a/crates/brk_traversable/src/lib.rs b/crates/brk_traversable/src/lib.rs index 5389b35ba..cba90737a 100644 --- a/crates/brk_traversable/src/lib.rs +++ b/crates/brk_traversable/src/lib.rs @@ -5,13 +5,13 @@ pub use brk_types::TreeNode; #[cfg(feature = "derive")] pub use brk_traversable_derive::Traversable; use vecdb::{ - AnyCollectableVec, AnyVec, CompressedVec, ComputedVec, EagerVec, LazyVecFrom1, LazyVecFrom2, + AnyVec, AnyWritableVec, CompressedVec, ComputedVec, EagerVec, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, RawVec, StoredCompressed, StoredIndex, StoredRaw, StoredVec, }; pub trait Traversable { fn to_tree_node(&self) -> TreeNode; - fn iter_any_collectable(&self) -> impl Iterator; + fn iter_any_writable(&self) -> impl Iterator; } impl Traversable for RawVec @@ -19,8 +19,8 @@ where I: StoredIndex, T: StoredRaw, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -33,8 +33,8 @@ where I: StoredIndex, T: StoredCompressed, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -47,8 +47,8 @@ where I: StoredIndex, T: StoredCompressed, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -61,8 +61,8 @@ where I: StoredIndex, T: StoredCompressed, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -77,8 +77,8 @@ where S1I: StoredIndex, S1T: StoredRaw, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -95,8 +95,8 @@ where S2I: StoredIndex, S2T: StoredRaw, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -116,8 +116,8 @@ where S3I: StoredIndex, S3T: StoredRaw, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -137,8 +137,8 @@ where S3I: StoredIndex, S3T: StoredCompressed, { - fn iter_any_collectable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyCollectableVec) + fn iter_any_writable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyWritableVec) } fn to_tree_node(&self) -> TreeNode { @@ -151,8 +151,8 @@ impl Traversable for Box { (**self).to_tree_node() } - fn iter_any_collectable(&self) -> impl Iterator { - (**self).iter_any_collectable() + fn iter_any_writable(&self) -> impl Iterator { + (**self).iter_any_writable() } } @@ -164,10 +164,11 @@ impl Traversable for Option { } } - fn iter_any_collectable(&self) -> impl Iterator { + fn iter_any_writable(&self) -> impl Iterator { match self { - Some(inner) => Box::new(inner.iter_any_collectable()) - as Box>, + Some(inner) => { + Box::new(inner.iter_any_writable()) as Box> + } None => Box::new(std::iter::empty()), } } @@ -182,11 +183,10 @@ impl Traversable for BTreeMap { TreeNode::Branch(children) } - fn iter_any_collectable(&self) -> impl Iterator { - let mut iter: Box> = - Box::new(std::iter::empty()); + fn iter_any_writable(&self) -> impl Iterator { + let mut iter: Box> = Box::new(std::iter::empty()); for v in self.values() { - iter = Box::new(iter.chain(v.iter_any_collectable())); + iter = Box::new(iter.chain(v.iter_any_writable())); } iter } diff --git a/crates/brk_traversable_derive/src/lib.rs b/crates/brk_traversable_derive/src/lib.rs index 19b9780e2..63cb69c94 100644 --- a/crates/brk_traversable_derive/src/lib.rs +++ b/crates/brk_traversable_derive/src/lib.rs @@ -29,8 +29,8 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { self.0.to_tree_node() } - fn iter_any_collectable(&self) -> impl Iterator { - self.0.iter_any_collectable() + fn iter_any_writable(&self) -> impl Iterator { + self.0.iter_any_writable() } } }); @@ -44,7 +44,7 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { brk_traversable::TreeNode::Branch(std::collections::BTreeMap::new()) } - fn iter_any_collectable(&self) -> impl Iterator { + fn iter_any_writable(&self) -> impl Iterator { std::iter::empty() } } @@ -281,7 +281,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { if regular_fields.is_empty() && option_fields.is_empty() { return quote! { - fn iter_any_collectable(&self) -> impl Iterator { + fn iter_any_writable(&self) -> impl Iterator { std::iter::empty() } }; @@ -290,17 +290,17 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { let (init_part, chain_part) = if let Some((&first, rest)) = regular_fields.split_first() { ( quote! { - let mut regular_iter: Box> = - Box::new(self.#first.iter_any_collectable()); + let mut regular_iter: Box> = + Box::new(self.#first.iter_any_writable()); }, quote! { - #(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_collectable()));)* + #(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_writable()));)* }, ) } else { ( quote! { - let mut regular_iter: Box> = + let mut regular_iter: Box> = Box::new(std::iter::empty()); }, quote! {}, @@ -311,7 +311,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { let chains = option_fields.iter().map(|f| { quote! { if let Some(ref x) = self.#f { - regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); } } }); @@ -321,7 +321,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { }; quote! { - fn iter_any_collectable(&self) -> impl Iterator { + fn iter_any_writable(&self) -> impl Iterator { #init_part #chain_part #option_part