Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-11-10 13:18:41 +01:00
parent f6a2a0540b
commit 37f5f50867
50 changed files with 1550 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

447
Cargo.lock generated
View File

@@ -506,6 +506,7 @@ dependencies = [
name = "brk"
version = "0.0.111"
dependencies = [
"brk_bencher",
"brk_binder",
"brk_bundler",
"brk_cli",
@@ -527,6 +528,20 @@ dependencies = [
"brk_types",
]
[[package]]
name = "brk_bencher"
version = "0.0.111"
dependencies = [
"brk_error",
]
[[package]]
name = "brk_bencher_visualizer"
version = "0.0.111"
dependencies = [
"plotters",
]
[[package]]
name = "brk_binder"
version = "0.0.111"
@@ -631,8 +646,6 @@ dependencies = [
[[package]]
name = "brk_fjall"
version = "2.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7d4da994dbf749f9ae892b1f447c7b0adda81ad16098d6bd9c3dd58be3d65b8"
dependencies = [
"byteorder",
"byteview 0.6.1",
@@ -661,6 +674,7 @@ name = "brk_indexer"
version = "0.0.111"
dependencies = [
"bitcoin",
"brk_bencher",
"brk_error",
"brk_fjall",
"brk_grouper",
@@ -673,7 +687,6 @@ dependencies = [
"brk_types",
"fjall",
"log",
"rand 0.9.2",
"rayon",
"rustc-hash",
"vecdb",
@@ -1303,6 +1316,12 @@ dependencies = [
"allocator-api2",
]
[[package]]
name = "bytemuck"
version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4"
[[package]]
name = "byteorder"
version = "1.5.0"
@@ -1458,6 +1477,12 @@ dependencies = [
"tracing-error",
]
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "colorchoice"
version = "1.0.4"
@@ -1529,12 +1554,58 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]]
name = "core-foundation"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core-graphics"
version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"core-graphics-types",
"foreign-types",
"libc",
]
[[package]]
name = "core-graphics-types"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"libc",
]
[[package]]
name = "core-text"
version = "20.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5"
dependencies = [
"core-foundation",
"core-graphics",
"foreign-types",
"libc",
]
[[package]]
name = "cow-utils"
version = "0.1.3"
@@ -1772,6 +1843,27 @@ dependencies = [
"crypto-common",
]
[[package]]
name = "dirs"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.61.2",
]
[[package]]
name = "dispatch2"
version = "0.3.0"
@@ -1795,6 +1887,15 @@ dependencies = [
"syn 2.0.109",
]
[[package]]
name = "dlib"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412"
dependencies = [
"libloading",
]
[[package]]
name = "double-ended-peekable"
version = "0.1.0"
@@ -1819,6 +1920,18 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dwrote"
version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b35532432acc8b19ceed096e35dfa088d3ea037fe4f3c085f1f97f33b4d02"
dependencies = [
"lazy_static",
"libc",
"winapi",
"wio",
]
[[package]]
name = "dyn-clone"
version = "1.0.20"
@@ -1936,6 +2049,15 @@ dependencies = [
"simdutf8",
]
[[package]]
name = "fdeflate"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
dependencies = [
"simd-adler32",
]
[[package]]
name = "filetime"
version = "0.2.26"
@@ -1962,16 +2084,15 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "fjall"
version = "3.0.0-pre.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf4746eb86124d4cd91564bc01c247b8c2c3ddf2d9fb3701ca16c86fdb5b538"
version = "3.0.0-pre.5"
dependencies = [
"byteorder-lite",
"byteview 0.8.0",
"dashmap",
"flume",
"log",
"lsm-tree 3.0.0-pre.5",
"std-semaphore",
"lz4_flex",
"tempfile",
"xxhash-rust",
]
@@ -1996,6 +2117,21 @@ dependencies = [
"num-traits",
]
[[package]]
name = "float-ord"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d"
[[package]]
name = "flume"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
dependencies = [
"spin",
]
[[package]]
name = "fnv"
version = "1.0.7"
@@ -2014,6 +2150,58 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "font-kit"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c7e611d49285d4c4b2e1727b72cf05353558885cc5252f93707b845dfcaf3d3"
dependencies = [
"bitflags 2.10.0",
"byteorder",
"core-foundation",
"core-graphics",
"core-text",
"dirs",
"dwrote",
"float-ord",
"freetype-sys",
"lazy_static",
"libc",
"log",
"pathfinder_geometry",
"pathfinder_simd",
"walkdir",
"winapi",
"yeslogic-fontconfig-sys",
]
[[package]]
name = "foreign-types"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
dependencies = [
"foreign-types-macros",
"foreign-types-shared",
]
[[package]]
name = "foreign-types-macros"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.109",
]
[[package]]
name = "foreign-types-shared"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -2023,6 +2211,17 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "freetype-sys"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7edc5b9669349acfda99533e9e0bcf26a51862ab43b08ee7745c55d28eb134"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "fsevent-sys"
version = "4.1.0"
@@ -2154,6 +2353,16 @@ dependencies = [
"wasip2",
]
[[package]]
name = "gif"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045"
dependencies = [
"color_quant",
"weezl",
]
[[package]]
name = "gimli"
version = "0.32.3"
@@ -2448,6 +2657,20 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "image"
version = "0.24.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"jpeg-decoder",
"num-traits",
"png",
]
[[package]]
name = "indenter"
version = "0.3.4"
@@ -2576,6 +2799,12 @@ dependencies = [
"libc",
]
[[package]]
name = "jpeg-decoder"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
[[package]]
name = "js-sys"
version = "0.3.82"
@@ -2645,6 +2874,16 @@ version = "0.2.177"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
[[package]]
name = "libloading"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
dependencies = [
"cfg-if",
"windows-link",
]
[[package]]
name = "libredox"
version = "0.1.10"
@@ -2719,8 +2958,6 @@ dependencies = [
[[package]]
name = "lsm-tree"
version = "3.0.0-pre.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af1cdfd737b63065d5162e39b581d6c9575990af0669cd200d5031bfd88adf25"
dependencies = [
"byteorder-lite",
"byteview 0.8.0",
@@ -2743,6 +2980,9 @@ name = "lz4_flex"
version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
dependencies = [
"twox-hash",
]
[[package]]
name = "matchit"
@@ -2987,6 +3227,12 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "outref"
version = "0.5.2"
@@ -3552,6 +3798,25 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
[[package]]
name = "pathfinder_geometry"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3"
dependencies = [
"log",
"pathfinder_simd",
]
[[package]]
name = "pathfinder_simd"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf9027960355bf3afff9841918474a81a5f972ac6d226d518060bba758b5ad57"
dependencies = [
"rustc_version",
]
[[package]]
name = "pco"
version = "0.4.7"
@@ -3663,6 +3928,65 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "plotters"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
dependencies = [
"chrono",
"font-kit",
"image",
"lazy_static",
"num-traits",
"pathfinder_geometry",
"plotters-backend",
"plotters-bitmap",
"plotters-svg",
"ttf-parser",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
[[package]]
name = "plotters-bitmap"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405"
dependencies = [
"gif",
"image",
"plotters-backend",
]
[[package]]
name = "plotters-svg"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
dependencies = [
"plotters-backend",
]
[[package]]
name = "png"
version = "0.17.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526"
dependencies = [
"bitflags 1.3.2",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide",
]
[[package]]
name = "pnp"
version = "0.12.5"
@@ -3930,6 +4254,17 @@ dependencies = [
"bitflags 2.10.0",
]
[[package]]
name = "redox_users"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
dependencies = [
"getrandom 0.2.16",
"libredox",
"thiserror 2.0.17",
]
[[package]]
name = "ref-cast"
version = "1.0.25"
@@ -4116,6 +4451,15 @@ version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustc_version"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "1.1.2"
@@ -4252,6 +4596,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16c2f82143577edb4921b71ede051dac62ca3c16084e918bf7b40c96ae10eb33"
[[package]]
name = "semver"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "seq-macro"
version = "0.3.6"
@@ -4502,6 +4852,15 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
dependencies = [
"lock_api",
]
[[package]]
name = "sse-stream"
version = "0.2.1"
@@ -5003,6 +5362,18 @@ dependencies = [
"termcolor",
]
[[package]]
name = "ttf-parser"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4"
[[package]]
name = "twox-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
[[package]]
name = "typedmap"
version = "0.6.0"
@@ -5269,12 +5640,44 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "web-sys"
version = "0.3.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.25.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
[[package]]
name = "weezl"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "009936b22a61d342859b5f0ea64681cbb35a358ab548e2a44a8cf0dac2d980b8"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.11"
@@ -5284,6 +5687,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.62.2"
@@ -5559,6 +5968,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "wio"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5"
dependencies = [
"winapi",
]
[[package]]
name = "wit-bindgen"
version = "0.46.0"
@@ -5583,6 +6001,17 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
[[package]]
name = "yeslogic-fontconfig-sys"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "503a066b4c037c440169d995b869046827dbc71263f6e8f3be6d77d4f3229dbd"
dependencies = [
"dlib",
"once_cell",
"pkg-config",
]
[[package]]
name = "yoke"
version = "0.8.1"

11
Cargo.toml
View File

@@ -37,6 +37,7 @@ aide = { version = "0.16.0-alpha.1", features = ["axum-json", "axum-query"] }
axum = "0.8.6"
bitcoin = { version = "0.32.7", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
brk_bencher = { version = "0.0.111", path = "crates/brk_bencher" }
brk_binder = { version = "0.0.111", path = "crates/brk_binder" }
brk_bundler = { version = "0.0.111", path = "crates/brk_bundler" }
brk_cli = { version = "0.0.111", path = "crates/brk_cli" }
@@ -60,12 +61,12 @@ brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_d
byteview = "=0.6.1"
# byteview = "~0.8.0"
derive_deref = "1.1.1"
fjall2 = { version = "2.11.5", package = "brk_fjall" }
# fjall2 = { path = "../fjall2", package = "brk_fjall" }
# fjall2 = { version = "2.11.5", package = "brk_fjall" }
fjall2 = { path = "../fjall2", package = "brk_fjall" }
# fjall2 = { version = "2.11.2", package = "fjall" }
fjall3 = { version = "=3.0.0-pre.4", package = "fjall" }
# fjall3 = { path = "../fjall3", package = "brk_fjall" }
# fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "bb15057500dce3115d7644d268b9deeaa895b431", package = "fjall" }
# fjall3 = { version = "=3.0.0-pre.5", package = "fjall" }
fjall3 = { path = "../fjall3", package = "fjall" }
# fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "f0bf96c2017b3543eb176012b8eff69c639dff1d", package = "fjall" }
jiff = "0.2.16"
log = "0.4.28"
minreq = { version = "2.14.1", features = ["https", "serde_json"] }

3
crates/brk/Cargo.toml
View File

@@ -12,6 +12,7 @@ build = "build.rs"
[features]
default = ["cli"]
full = [
"bencher",
"binder",
"bundler",
"cli",
@@ -32,6 +33,7 @@ full = [
"traversable",
"types",
]
bencher = ["brk_bencher"]
binder = ["brk_binder"]
bundler = ["brk_bundler"]
cli = ["brk_cli"]
@@ -53,6 +55,7 @@ traversable = ["brk_traversable"]
types = ["brk_types"]
[dependencies]
brk_bencher = { workspace = true, optional = true }
brk_binder = { workspace = true, optional = true }
brk_bundler = { workspace = true, optional = true }
brk_cli = { workspace = true, optional = true }

4
crates/brk/src/lib.rs
View File

@@ -1,5 +1,9 @@
#![doc = include_str!("../README.md")]
#[cfg(feature = "bencher")]
#[doc(inline)]
pub use brk_bencher as bencher;
#[cfg(feature = "binder")]
#[doc(inline)]
pub use brk_binder as binder;

13
crates/brk_bencher/Cargo.toml Normal file
View File

@@ -0,0 +1,13 @@
[package]
name = "brk_bencher"
description = "A simple benchmarker for testing other crates."
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
rust-version.workspace = true
build = "build.rs"
[dependencies]
brk_error = { workspace = true }

1
crates/brk_bencher/README.md Normal file
View File

@@ -0,0 +1 @@
# brk_bencher

8
crates/brk_bencher/build.rs Normal file
View File

@@ -0,0 +1,8 @@
fn main() {
let profile = std::env::var("PROFILE").unwrap_or_default();
if profile == "release" {
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-cpu=native");
}
}

233
crates/brk_bencher/src/lib.rs Normal file
View File

@@ -0,0 +1,233 @@
use std::{
fs,
io::Write,
path::{Path, PathBuf},
process::Command,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
thread::{self, JoinHandle},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use brk_error::Result;
pub struct Bencher {
bench_dir: PathBuf,
stop_flag: Arc<AtomicBool>,
monitor_thread: Option<JoinHandle<Result<()>>>,
}
impl Bencher {
/// Create a new bencher for the given crate name
/// Creates directory structure: workspace_root/benches/{crate_name}/{timestamp}/
pub fn new(crate_name: &str, workspace_root: &Path) -> Result<Self> {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
let bench_dir = workspace_root
.join("benches")
.join(crate_name)
.join(timestamp.to_string());
fs::create_dir_all(&bench_dir)?;
Ok(Self {
bench_dir,
stop_flag: Arc::new(AtomicBool::new(false)),
monitor_thread: None,
})
}
/// Create a bencher using CARGO_MANIFEST_DIR to find workspace root
pub fn from_cargo_env() -> Result<Self> {
let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.ok_or("Failed to find workspace root")?;
let crate_name = env!("CARGO_PKG_NAME");
Self::new(crate_name, workspace_root)
}
/// Start monitoring disk usage and memory footprint
pub fn start(&mut self) -> Result<()> {
if self.monitor_thread.is_some() {
return Err("Bencher already started".into());
}
let stop_flag = self.stop_flag.clone();
let bench_dir = self.bench_dir.clone();
let handle = thread::spawn(move || monitor_resources(&bench_dir, stop_flag));
self.monitor_thread = Some(handle);
Ok(())
}
/// Stop monitoring and wait for the thread to finish
pub fn stop(mut self) -> Result<()> {
self.stop_flag.store(true, Ordering::Relaxed);
if let Some(handle) = self.monitor_thread.take() {
handle.join().map_err(|_| "Monitor thread panicked")??;
}
Ok(())
}
/// Get the benchmark output directory
pub fn bench_dir(&self) -> &Path {
&self.bench_dir
}
}
fn parse_size_to_mb(value_str: &str, unit: &str) -> Option<f64> {
let value: f64 = value_str.parse().ok()?;
match unit {
"MB" | "M" => Some(value),
"GB" | "G" => Some(value * 1024.0),
"KB" | "K" => Some(value / 1024.0),
"B" => Some(value / 1024.0 / 1024.0),
_ => None,
}
}
fn parse_du_output(size_str: &str) -> Option<f64> {
// Parse outputs like "524M", "287G", "4.0K"
let size_str = size_str.trim();
if let Some(unit_pos) = size_str.find(|c: char| c.is_alphabetic()) {
let (value_part, unit_part) = size_str.split_at(unit_pos);
parse_size_to_mb(value_part, unit_part)
} else {
// No unit means bytes
let value: f64 = size_str.parse().ok()?;
Some(value / 1024.0 / 1024.0)
}
}
fn parse_footprint_output(output: &str) -> Option<(f64, f64)> {
let mut phys_footprint = None;
let mut phys_footprint_peak = None;
for line in output.lines() {
if line.contains("phys_footprint:") && !line.contains("peak") {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 3 {
phys_footprint = parse_size_to_mb(parts[1], parts[2]);
}
} else if line.contains("phys_footprint_peak:") {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 3 {
phys_footprint_peak = parse_size_to_mb(parts[1], parts[2]);
}
}
}
match (phys_footprint, phys_footprint_peak) {
(Some(f), Some(p)) => Some((f, p)),
_ => None,
}
}
#[cfg(target_os = "linux")]
fn get_memory_usage_linux(pid: u32) -> Result<(f64, f64)> {
// Read /proc/[pid]/status for memory information
let status_path = format!("/proc/{}/status", pid);
let status_content = fs::read_to_string(status_path)?;
let mut vm_rss = None;
let mut vm_hwm = None;
for line in status_content.lines() {
if line.starts_with("VmRSS:") {
// Current RSS in kB
if let Some(value_str) = line.split_whitespace().nth(1)
&& let Ok(kb) = value_str.parse::<f64>()
{
vm_rss = Some(kb / 1024.0); // Convert kB to MB
}
} else if line.starts_with("VmHWM:") {
// Peak RSS (High Water Mark) in kB
if let Some(value_str) = line.split_whitespace().nth(1)
&& let Ok(kb) = value_str.parse::<f64>()
{
vm_hwm = Some(kb / 1024.0); // Convert kB to MB
}
}
}
match (vm_rss, vm_hwm) {
(Some(rss), Some(hwm)) => Ok((rss, hwm)),
_ => Err("Failed to parse memory info from /proc/[pid]/status".into()),
}
}
#[cfg(target_os = "macos")]
fn get_memory_usage_macos(pid: u32) -> Result<(f64, f64)> {
let output = Command::new("footprint")
.args(["-p", &pid.to_string()])
.output()?;
let stdout = String::from_utf8(output.stdout).unwrap();
parse_footprint_output(&stdout).ok_or_else(|| "Failed to parse footprint output".into())
}
fn get_memory_usage(pid: u32) -> Result<(f64, f64)> {
#[cfg(target_os = "macos")]
{
get_memory_usage_macos(pid)
}
#[cfg(target_os = "linux")]
{
get_memory_usage_linux(pid)
}
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
{
Err("Unsupported platform for memory monitoring".into())
}
}
fn monitor_resources(bench_dir: &Path, stop_flag: Arc<AtomicBool>) -> Result<()> {
let disk_file = bench_dir.join("disk_usage.csv");
let memory_file = bench_dir.join("memory_footprint.csv");
let mut disk_writer = fs::File::create(disk_file)?;
let mut memory_writer = fs::File::create(memory_file)?;
writeln!(disk_writer, "timestamp_ms,disk_usage_mb")?;
writeln!(
memory_writer,
"timestamp_ms,phys_footprint_mb,phys_footprint_peak_mb"
)?;
let pid = std::process::id();
let start = Instant::now();
while !stop_flag.load(Ordering::Relaxed) {
let elapsed_ms = start.elapsed().as_millis();
// Get disk usage
if let Ok(output) = Command::new("du")
.args(["-sh", bench_dir.to_str().unwrap()])
.output()
&& let Ok(stdout) = String::from_utf8(output.stdout)
&& let Some(size_str) = stdout.split_whitespace().next()
&& let Some(size_mb) = parse_du_output(size_str)
{
writeln!(disk_writer, "{},{}", elapsed_ms, size_mb)?;
disk_writer.flush()?;
}
// Get memory footprint (cross-platform)
if let Ok((footprint, peak)) = get_memory_usage(pid) {
writeln!(memory_writer, "{},{},{}", elapsed_ms, footprint, peak)?;
memory_writer.flush()?;
}
thread::sleep(Duration::from_secs(1));
}
Ok(())
}

13
crates/brk_bencher_visualizer/Cargo.toml Normal file
View File

@@ -0,0 +1,13 @@
[package]
name = "brk_bencher_visualizer"
description = "A generator of charts for brk_bencher"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
rust-version.workspace = true
build = "build.rs"
[dependencies]
plotters = "0.3.7"

0
crates/brk_bencher_visualizer/README.md Normal file
View File

8
crates/brk_bencher_visualizer/build.rs Normal file
View File

@@ -0,0 +1,8 @@
fn main() {
let profile = std::env::var("PROFILE").unwrap_or_default();
if profile == "release" {
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-cpu=native");
}
}

317
crates/brk_bencher_visualizer/src/lib.rs Normal file
View File

@@ -0,0 +1,317 @@
use plotters::prelude::*;
use std::{
fs,
path::{Path, PathBuf},
};
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
#[derive(Debug, Clone)]
struct DataPoint {
timestamp_ms: u64,
value: f64,
}
#[derive(Debug)]
struct BenchmarkRun {
run_id: String,
data: Vec<DataPoint>,
}
pub struct Visualizer {
workspace_root: PathBuf,
}
impl Visualizer {
pub fn new(workspace_root: impl AsRef<Path>) -> Self {
Self {
workspace_root: workspace_root.as_ref().to_path_buf(),
}
}
pub fn from_cargo_env() -> Result<Self> {
let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.ok_or("Failed to find workspace root")?
.to_path_buf();
Ok(Self { workspace_root })
}
/// Generate all charts for all crates in the benches directory
pub fn generate_all_charts(&self) -> Result<()> {
let benches_dir = self.workspace_root.join("benches");
if !benches_dir.exists() {
return Err("Benches directory does not exist".into());
}
// Iterate through each crate directory
for entry in fs::read_dir(&benches_dir)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
let crate_name = path
.file_name()
.and_then(|n| n.to_str())
.ok_or("Invalid crate name")?;
println!("Generating charts for crate: {}", crate_name);
self.generate_crate_charts(&path, crate_name)?;
}
}
Ok(())
}
/// Generate charts for a specific crate
fn generate_crate_charts(&self, crate_path: &Path, crate_name: &str) -> Result<()> {
// Read all benchmark runs for this crate
let disk_runs = self.read_benchmark_runs(crate_path, "disk_usage.csv")?;
let memory_runs = self.read_benchmark_runs(crate_path, "memory_footprint.csv")?;
if !disk_runs.is_empty() {
self.generate_disk_chart(crate_path, crate_name, &disk_runs)?;
}
if !memory_runs.is_empty() {
self.generate_memory_chart(crate_path, crate_name, &memory_runs)?;
}
Ok(())
}
/// Read all benchmark runs from subdirectories
fn read_benchmark_runs(&self, crate_path: &Path, filename: &str) -> Result<Vec<BenchmarkRun>> {
let mut runs = Vec::new();
for entry in fs::read_dir(crate_path)? {
let entry = entry?;
let run_path = entry.path();
if run_path.is_dir() {
let run_id = run_path
.file_name()
.and_then(|n| n.to_str())
.ok_or("Invalid run ID")?
.to_string();
let csv_path = run_path.join(filename);
if csv_path.exists()
&& let Ok(data) = self.read_csv(&csv_path)
{
runs.push(BenchmarkRun { run_id, data });
}
}
}
Ok(runs)
}
/// Read a CSV file and parse data points
fn read_csv(&self, path: &Path) -> Result<Vec<DataPoint>> {
let content = fs::read_to_string(path)?;
let mut data = Vec::new();
for (i, line) in content.lines().enumerate() {
if i == 0 {
continue;
} // Skip header
let parts: Vec<&str> = line.split(',').collect();
if parts.len() >= 2
&& let (Ok(timestamp_ms), Ok(value)) =
(parts[0].parse::<u64>(), parts[1].parse::<f64>())
{
data.push(DataPoint {
timestamp_ms,
value,
});
}
}
Ok(data)
}
/// Generate disk usage chart
fn generate_disk_chart(
&self,
crate_path: &Path,
crate_name: &str,
runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("disk_usage_chart.png");
let root = BitMapBackend::new(&output_path, (1200, 800)).into_drawing_area();
root.fill(&WHITE)?;
let max_time = runs
.iter()
.flat_map(|r| r.data.iter().map(|d| d.timestamp_ms))
.max()
.unwrap_or(1000);
let max_value = runs
.iter()
.flat_map(|r| r.data.iter().map(|d| d.value))
.fold(0.0_f64, f64::max);
let mut chart = ChartBuilder::on(&root)
.caption(
format!("{} - Disk Usage", crate_name),
("sans-serif", 40).into_font(),
)
.margin(10)
.x_label_area_size(40)
.y_label_area_size(60)
.build_cartesian_2d(0u64..max_time, 0.0..max_value * 1.1)?;
chart
.configure_mesh()
.x_desc("Time (ms)")
.y_desc("Disk Usage (MB)")
.draw()?;
let colors = [&RED, &BLUE, &GREEN, &CYAN, &MAGENTA, &YELLOW];
for (idx, run) in runs.iter().enumerate() {
let color = colors[idx % colors.len()];
chart
.draw_series(LineSeries::new(
run.data.iter().map(|d| (d.timestamp_ms, d.value)),
color,
))?
.label(&run.run_id)
.legend(move |(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], color));
}
chart
.configure_series_labels()
.background_style(WHITE.mix(0.8))
.border_style(BLACK)
.draw()?;
root.present()?;
println!("Generated: {}", output_path.display());
Ok(())
}
/// Generate memory footprint chart
fn generate_memory_chart(
&self,
crate_path: &Path,
crate_name: &str,
runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("memory_footprint_chart.png");
let root = BitMapBackend::new(&output_path, (1200, 800)).into_drawing_area();
root.fill(&WHITE)?;
// Read memory CSV files which have 3 columns: timestamp, footprint, peak
let mut enhanced_runs = Vec::new();
for run in runs {
// Re-read the CSV to get both footprint and peak values
let csv_path = crate_path.join(&run.run_id).join("memory_footprint.csv");
if let Ok(content) = fs::read_to_string(&csv_path) {
let mut footprint_data = Vec::new();
let mut peak_data = Vec::new();
for (i, line) in content.lines().enumerate() {
if i == 0 {
continue;
} // Skip header
let parts: Vec<&str> = line.split(',').collect();
if parts.len() >= 3
&& let (Ok(timestamp_ms), Ok(footprint), Ok(peak)) = (
parts[0].parse::<u64>(),
parts[1].parse::<f64>(),
parts[2].parse::<f64>(),
)
{
footprint_data.push(DataPoint {
timestamp_ms,
value: footprint,
});
peak_data.push(DataPoint {
timestamp_ms,
value: peak,
});
}
}
enhanced_runs.push((run.run_id.clone(), footprint_data, peak_data));
}
}
let max_time = enhanced_runs
.iter()
.flat_map(|(_, f, p)| f.iter().chain(p.iter()).map(|d| d.timestamp_ms))
.max()
.unwrap_or(1000);
let max_value = enhanced_runs
.iter()
.flat_map(|(_, f, p)| f.iter().chain(p.iter()).map(|d| d.value))
.fold(0.0_f64, f64::max);
let mut chart = ChartBuilder::on(&root)
.caption(
format!("{} - Memory Footprint", crate_name),
("sans-serif", 40).into_font(),
)
.margin(10)
.x_label_area_size(40)
.y_label_area_size(60)
.build_cartesian_2d(0u64..max_time, 0.0..max_value * 1.1)?;
chart
.configure_mesh()
.x_desc("Time (ms)")
.y_desc("Memory (MB)")
.draw()?;
let colors = [&RED, &BLUE, &GREEN, &CYAN, &MAGENTA, &YELLOW];
for (idx, (run_id, footprint_data, peak_data)) in enhanced_runs.iter().enumerate() {
let color = colors[idx % colors.len()];
// Draw footprint line (solid)
chart
.draw_series(LineSeries::new(
footprint_data.iter().map(|d| (d.timestamp_ms, d.value)),
color,
))?
.label(format!("{} (current)", run_id))
.legend(move |(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], color));
// Draw peak line (dashed)
chart
.draw_series(LineSeries::new(
peak_data.iter().map(|d| (d.timestamp_ms, d.value)),
color.stroke_width(2).filled(),
))?
.label(format!("{} (peak)", run_id))
.legend(move |(x, y)| {
PathElement::new(
vec![(x, y), (x + 10, y), (x + 20, y)],
color.stroke_width(2),
)
});
}
chart
.configure_series_labels()
.background_style(WHITE.mix(0.8))
.border_style(BLACK)
.draw()?;
root.present()?;
println!("Generated: {}", output_path.display());
Ok(())
}
}

6
crates/brk_bencher_visualizer/src/main.rs Normal file
View File

@@ -0,0 +1,6 @@
use brk_bencher_visualizer::Visualizer;
fn main() {
let v = Visualizer::from_cargo_env().unwrap();
v.generate_all_charts().unwrap();
}

2
crates/brk_computer/src/blks.rs
View File

@@ -43,7 +43,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/chain.rs
View File

@@ -1097,7 +1097,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/cointime.rs
View File

@@ -279,7 +279,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/constants.rs
View File

@@ -162,7 +162,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/fetched.rs
View File

@@ -44,7 +44,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/grouped/builder_eager.rs
View File

@@ -534,7 +534,7 @@ where
pub fn starting_index(&self, max_from: I) -> I {
max_from.min(I::from(
self.iter_any_collectable().map(|v| v.len()).min().unwrap(),
self.iter_any_writable().map(|v| v.len()).min().unwrap(),
))
}

2
crates/brk_computer/src/grouped/builder_lazy.rs
View File

@@ -209,7 +209,7 @@ where
pub fn starting_index(&self, max_from: I) -> I {
max_from.min(I::from(
self.iter_any_collectable().map(|v| v.len()).min().unwrap(),
self.iter_any_writable().map(|v| v.len()).min().unwrap(),
))
}

22
crates/brk_computer/src/grouped/from_dateindex.rs
View File

@@ -4,7 +4,7 @@ use brk_traversable::Traversable;
use brk_types::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex,
};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec};
use vecdb::{AnyWritableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec};
use crate::{Indexes, grouped::LazyVecsBuilder, indexes};
@@ -178,17 +178,17 @@ where
.unwrap()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(self.dateindex_extra.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable()));
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyWritableVec>> =
Box::new(self.dateindex_extra.iter_any_writable());
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable()));
if let Some(ref x) = self.dateindex {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(x.iter_any_writable()));
}
regular_iter
}

26
crates/brk_computer/src/grouped/from_height.rs
View File

@@ -5,7 +5,7 @@ use brk_types::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec};
use vecdb::{AnyWritableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec};
use crate::{
Indexes,
@@ -239,19 +239,19 @@ where
.unwrap()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(self.height_extra.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable()));
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyWritableVec>> =
Box::new(self.height_extra.iter_any_writable());
regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable()));
if let Some(ref x) = self.height {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(x.iter_any_writable()));
}
regular_iter
}

12
crates/brk_computer/src/grouped/from_height_strict.rs
View File

@@ -2,7 +2,7 @@ use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DifficultyEpoch, Height, Version};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit};
use vecdb::{AnyWritableVec, Database, EagerVec, Exit};
use crate::{Indexes, indexes};
@@ -110,11 +110,11 @@ where
.merge_branches()
.unwrap()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(self.height.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable()));
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyWritableVec>> =
Box::new(self.height.iter_any_writable());
regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable()));
regular_iter
}
}

26
crates/brk_computer/src/grouped/from_txindex.rs
View File

@@ -6,7 +6,7 @@ use brk_types::{
Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCollectableVec, AnyVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec,
AnyVec, AnyWritableVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec,
IterableCloneableVec, StoredIndex, TypedVecIterator,
};
@@ -521,19 +521,19 @@ where
.unwrap()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(self.height.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable()));
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyWritableVec>> =
Box::new(self.height.iter_any_writable());
regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable()));
if let Some(ref x) = self.txindex {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(x.iter_any_writable()));
}
regular_iter
}

2
crates/brk_computer/src/indexes.rs
View File

@@ -492,7 +492,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/market.rs
View File

@@ -1517,7 +1517,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/pools/mod.rs
View File

@@ -62,7 +62,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/price.rs
View File

@@ -327,7 +327,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

2
crates/brk_computer/src/stateful/mod.rs
View File

@@ -482,7 +482,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

16
crates/brk_grouper/src/address.rs
View File

@@ -1,6 +1,6 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use vecdb::AnyCollectableVec;
use vecdb::AnyWritableVec;
use crate::Filtered;
@@ -78,14 +78,14 @@ where
)
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
[
Box::new(self.ge_amount.iter_any_collectable())
as Box<dyn Iterator<Item = &dyn AnyCollectableVec>>,
Box::new(self.amount_range.iter_any_collectable())
as Box<dyn Iterator<Item = &dyn AnyCollectableVec>>,
Box::new(self.lt_amount.iter_any_collectable())
as Box<dyn Iterator<Item = &dyn AnyCollectableVec>>,
Box::new(self.ge_amount.iter_any_writable())
as Box<dyn Iterator<Item = &dyn AnyWritableVec>>,
Box::new(self.amount_range.iter_any_writable())
as Box<dyn Iterator<Item = &dyn AnyWritableVec>>,
Box::new(self.lt_amount.iter_any_writable())
as Box<dyn Iterator<Item = &dyn AnyWritableVec>>,
]
.into_iter()
.flatten()

22
crates/brk_grouper/src/by_address_type.rs
View File

@@ -4,7 +4,7 @@ use brk_error::Result;
use brk_traversable::{Traversable, TreeNode};
use brk_types::OutputType;
use rayon::prelude::*;
use vecdb::AnyCollectableVec;
use vecdb::AnyWritableVec;
use super::{Filter, Filtered};
@@ -308,16 +308,16 @@ impl<T: Traversable> Traversable for ByAddressType<T> {
)
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(self.p2pk65.iter_any_collectable());
iter = Box::new(iter.chain(self.p2pk33.iter_any_collectable()));
iter = Box::new(iter.chain(self.p2pkh.iter_any_collectable()));
iter = Box::new(iter.chain(self.p2sh.iter_any_collectable()));
iter = Box::new(iter.chain(self.p2wpkh.iter_any_collectable()));
iter = Box::new(iter.chain(self.p2wsh.iter_any_collectable()));
iter = Box::new(iter.chain(self.p2tr.iter_any_collectable()));
iter = Box::new(iter.chain(self.p2a.iter_any_collectable()));
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
let mut iter: Box<dyn Iterator<Item = &dyn AnyWritableVec>> =
Box::new(self.p2pk65.iter_any_writable());
iter = Box::new(iter.chain(self.p2pk33.iter_any_writable()));
iter = Box::new(iter.chain(self.p2pkh.iter_any_writable()));
iter = Box::new(iter.chain(self.p2sh.iter_any_writable()));
iter = Box::new(iter.chain(self.p2wpkh.iter_any_writable()));
iter = Box::new(iter.chain(self.p2wsh.iter_any_writable()));
iter = Box::new(iter.chain(self.p2tr.iter_any_writable()));
iter = Box::new(iter.chain(self.p2a.iter_any_writable()));
iter
}
}

6
crates/brk_grouper/src/filter.rs
View File

@@ -2,7 +2,7 @@ use std::ops::Range;
use brk_traversable::{Traversable, TreeNode};
use brk_types::{HalvingEpoch, OutputType};
use vecdb::AnyCollectableVec;
use vecdb::AnyWritableVec;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Filter {
@@ -82,7 +82,7 @@ impl<T: Traversable> Traversable for Filtered<T> {
self.1.to_tree_node()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
self.1.iter_any_collectable()
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
self.1.iter_any_writable()
}
}

2
crates/brk_indexer/Cargo.toml
View File

@@ -11,6 +11,7 @@ build = "build.rs"
[dependencies]
bitcoin = { workspace = true }
brk_bencher = { workspace = true }
brk_error = { workspace = true }
brk_grouper = { workspace = true }
brk_iterator = { workspace = true }
@@ -26,4 +27,3 @@ log = { workspace = true }
rayon = { workspace = true }
rustc-hash = { workspace = true }
vecdb = { workspace = true }
rand = "0.9.2"

4
crates/brk_indexer/examples/indexer.rs
View File

@@ -10,6 +10,7 @@ use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_reader::Reader;
use brk_rpc::{Auth, Client};
use log::debug;
use vecdb::Exit;
fn main() -> Result<()> {
@@ -28,12 +29,15 @@ fn main() -> Result<()> {
)?;
let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
debug!("Reader created.");
let blocks = Blocks::new(&client, &reader);
debug!("Blocks created.");
fs::create_dir_all(&outputs_dir)?;
let mut indexer = Indexer::forced_import(&outputs_dir)?;
debug!("Indexer imported.");
let exit = Exit::new();
exit.set_ctrlc_handler();

48
crates/brk_indexer/examples/indexer_bench.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{env, fs, path::Path, time::Instant};
use brk_bencher::Bencher;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_reader::Reader;
use brk_rpc::{Auth, Client};
use vecdb::Exit;
fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")))?;
let bitcoin_dir = Client::default_bitcoin_path();
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
fs::create_dir_all(&outputs_dir)?;
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let client = Client::new(
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;
let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
let blocks = Blocks::new(&client, &reader);
fs::create_dir_all(&outputs_dir)?;
let mut indexer = Indexer::forced_import(&outputs_dir)?;
let exit = Exit::new();
exit.set_ctrlc_handler();
let mut bencher = Bencher::from_cargo_env()?;
bencher.start()?;
let i = Instant::now();
indexer.checked_index(&blocks, &client, &exit)?;
dbg!(i.elapsed());
// Stop and finalize
bencher.stop()?;
Ok(())
}

3
crates/brk_indexer/src/indexes.rs
View File

@@ -5,6 +5,7 @@ use brk_types::{
P2WPKHAddressIndex, P2WSHAddressIndex, TxInIndex, TxIndex, TxOutIndex, TypeIndex,
UnknownOutputIndex,
};
use log::debug;
use vecdb::{GenericStoredVec, IterableStoredVec, IterableVec, StoredIndex, StoredRaw};
use crate::{Stores, Vecs};
@@ -88,6 +89,8 @@ impl Indexes {
impl From<(Height, &mut Vecs, &Stores)> for Indexes {
#[inline]
fn from((min_height, vecs, stores): (Height, &mut Vecs, &Stores)) -> Self {
debug!("Creating indexes from vecs and stores...");
// Height at which we want to start: min last saved + 1 or 0
let vecs_starting_height = vecs.starting_height();
let stores_starting_height = stores.starting_height();

10
crates/brk_indexer/src/lib.rs
View File

@@ -12,7 +12,7 @@ use brk_types::{
OutPoint, OutputType, Sats, StoredBool, Timestamp, TxInIndex, TxIndex, TxOutIndex, Txid,
TxidPrefix, TypeIndex, Unit, Version, Vin, Vout,
};
use log::{error, info};
use log::{debug, error, info};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, TypedVecIterator};
@@ -80,7 +80,11 @@ impl Indexer {
exit: &Exit,
check_collisions: bool,
) -> Result<Indexes> {
debug!("Starting indexing...");
let last_blockhash = self.vecs.height_to_blockhash.iter()?.last();
debug!("Last block hash found.");
let (starting_indexes, prev_hash) = if let Some(hash) = last_blockhash {
let (height, hash) = client.get_closest_valid_height(hash)?;
let starting_indexes =
@@ -93,15 +97,19 @@ impl Indexer {
} else {
(Indexes::default(), None)
};
info!("Starting indexes set.");
let lock = exit.lock();
self.stores
.rollback_if_needed(&mut self.vecs, &starting_indexes)?;
debug!("Rollback stores done.");
self.vecs.rollback_if_needed(&starting_indexes)?;
debug!("Rollback vecs done.");
drop(lock);
// Cloned because we want to return starting indexes for the computer
let mut indexes = starting_indexes.clone();
debug!("Indexes cloned.");
let should_export = |height: Height, rem: bool| -> bool {
height != 0 && (height % SNAPSHOT_BLOCK_RANGE == 0) != rem

51
crates/brk_indexer/src/stores_v2.rs
View File

@@ -20,12 +20,8 @@ use super::Vecs;
pub struct Stores {
pub keyspace: TransactionalKeyspace,
// pub addresshash_to_typeindex: Store<AddressHash, TypeIndex>,
pub addresstype_to_addresshash_to_addressindex: ByAddressType<Store<AddressHash, TypeIndex>>,
// pub addresstype_to_addressindex_and_txindex: Store<AddressTypeAddressIndexTxIndex, Unit>,
pub addresstype_to_addressindex_and_txindex: ByAddressType<Store<AddressIndexTxIndex, Unit>>,
// pub addresstype_to_addressindex_and_unspentoutpoint:
// Store<AddressTypeAddressIndexOutPoint, Unit>,
pub addresstype_to_addressindex_and_unspentoutpoint:
ByAddressType<Store<AddressIndexOutPoint, Unit>>,
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
@@ -94,14 +90,6 @@ impl Stores {
Mode::UniquePushOnly(Type::Sequential),
Compression::Lz4,
)?,
// addresshash_to_typeindex: Store::import(
// keyspace_ref,
// path,
// "a2t",
// version,
// Mode::UniquePushOnly(Type::Random),
// Compression::Lz4,
// )?,
addresstype_to_addresshash_to_addressindex: ByAddressType::new_with_index(
create_addresshash_to_addressindex_store,
)?,
@@ -121,25 +109,9 @@ impl Stores {
Mode::UniquePushOnly(Type::Random),
Compression::Lz4,
)?,
// addresstype_to_addressindex_and_txindex: Store::import(
// keyspace_ref,
// path,
// "aat",
// version,
// Mode::VecLike,
// Compression::Lz4,
// )?,
addresstype_to_addressindex_and_txindex: ByAddressType::new_with_index(
create_addressindex_to_txindex_store,
)?,
// addresstype_to_addressindex_and_unspentoutpoint: Store::import(
// keyspace_ref,
// path,
// "aau",
// version,
// Mode::VecLike,
// Compression::Lz4,
// )?,
addresstype_to_addressindex_and_unspentoutpoint: ByAddressType::new_with_index(
create_addressindex_to_unspentoutpoint_store,
)?,
@@ -158,13 +130,10 @@ impl Stores {
}
pub fn commit(&mut self, height: Height) -> Result<()> {
[
let tuples = [
&mut self.blockhashprefix_to_height as &mut dyn AnyStore,
&mut self.height_to_coinbase_tag,
&mut self.txidprefix_to_txindex,
// &mut self.addresshash_to_typeindex
// &mut self.addresstype_to_addressindex_and_txindex,
// &mut self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_par_iter()
.chain(
@@ -182,7 +151,15 @@ impl Stores {
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
) // Changed from par_iter_mut()
.try_for_each(|store| store.commit(height))?;
.map(|store| {
let items = store.take_all_f2();
store.export_meta_if_needed(height)?;
Ok((store.partition(), items))
})
.collect::<Result<Vec<_>>>()?;
let batch = self.keyspace.inner().batch();
batch.commit_partitions(tuples)?;
self.keyspace
.persist(PersistMode::SyncAll)
@@ -194,9 +171,6 @@ impl Stores {
&self.blockhashprefix_to_height as &dyn AnyStore,
&self.height_to_coinbase_tag,
&self.txidprefix_to_txindex,
// &self.addresshash_to_typeindex,
// &self.addresstype_to_addressindex_and_txindex,
// &self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_iter()
.chain(
@@ -224,11 +198,6 @@ impl Stores {
if self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?
&& self.height_to_coinbase_tag.is_empty()?
// && self.addresshash_to_typeindex.is_empty()?
// && self.addresstype_to_addressindex_and_txindex.is_empty()?
// && self
// .addresstype_to_addressindex_and_unspentoutpoint
// .is_empty()?
&& self
.addresstype_to_addresshash_to_addressindex
.iter()

266
crates/brk_indexer/src/stores_v3.rs
View File

@@ -1,11 +1,12 @@
use std::{fs, path::Path};
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_store::{AnyStore, Kind3, Mode3, StoreFjallV3 as Store};
use brk_types::{
AddressBytes, AddressHash, AddressTypeAddressIndexOutPoint, AddressTypeAddressIndexTxIndex,
BlockHashPrefix, Height, OutPoint, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex,
Unit, Version, Vout,
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height,
OutPoint, OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version,
Vout,
};
use fjall3::{Database, PersistMode};
use rayon::prelude::*;
@@ -19,11 +20,15 @@ use super::Vecs;
pub struct Stores {
pub database: Database,
pub addresshash_to_typeindex: Store<AddressHash, TypeIndex>,
pub addresstype_to_addresshash_to_addressindex: ByAddressType<Store<AddressHash, TypeIndex>>,
pub addresstype_to_addressindex_and_txindex: ByAddressType<Store<AddressIndexTxIndex, Unit>>,
pub addresstype_to_addressindex_and_unspentoutpoint:
ByAddressType<Store<AddressIndexOutPoint, Unit>>,
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
pub height_to_coinbase_tag: Store<Height, StoredString>,
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
pub addresstype_to_addressindex_and_txindex: Store<AddressTypeAddressIndexTxIndex, Unit>,
// pub addresstype_to_addressindex_and_txindex: Store<AddressTypeAddressIndexTxIndex, Unit>,
// pub addresshash_to_typeindex: Store<AddressHash, TypeIndex>,
// pub addresstype_to_addressindex_and_unspentoutpoint:
// Store<AddressTypeAddressIndexOutPoint, Unit>,
}
@@ -45,6 +50,39 @@ impl Stores {
let database_ref = &database;
let create_addresshash_to_addressindex_store = |index| {
Store::import(
database_ref,
path,
&format!("h2i{}", index),
version,
Mode3::PushOnly,
Kind3::Random,
)
};
let create_addressindex_to_txindex_store = |index| {
Store::import(
database_ref,
path,
&format!("a2t{}", index),
version,
Mode3::PushOnly,
Kind3::Vec,
)
};
let create_addressindex_to_unspentoutpoint_store = |index| {
Store::import(
database_ref,
path,
&format!("a2u{}", index),
version,
Mode3::Any,
Kind3::Vec,
)
};
Ok(Self {
database: database.clone(),
@@ -56,14 +94,23 @@ impl Stores {
Mode3::PushOnly,
Kind3::Sequential,
)?,
addresshash_to_typeindex: Store::import(
database_ref,
path,
"addresshash_to_typeindex",
version,
Mode3::PushOnly,
Kind3::Random,
addresstype_to_addresshash_to_addressindex: ByAddressType::new_with_index(
create_addresshash_to_addressindex_store,
)?,
addresstype_to_addressindex_and_txindex: ByAddressType::new_with_index(
create_addressindex_to_txindex_store,
)?,
addresstype_to_addressindex_and_unspentoutpoint: ByAddressType::new_with_index(
create_addressindex_to_unspentoutpoint_store,
)?,
// addresshash_to_typeindex: Store::import(
// database_ref,
// path,
// "addresshash_to_typeindex",
// version,
// Mode3::PushOnly,
// Kind3::Random,
// )?,
blockhashprefix_to_height: Store::import(
database_ref,
path,
@@ -80,14 +127,14 @@ impl Stores {
Mode3::PushOnly,
Kind3::Random,
)?,
addresstype_to_addressindex_and_txindex: Store::import(
database_ref,
path,
"addresstype_to_addressindex_and_txindex",
version,
Mode3::PushOnly,
Kind3::Vec,
)?,
// addresstype_to_addressindex_and_txindex: Store::import(
// database_ref,
// path,
// "addresstype_to_addressindex_and_txindex",
// version,
// Mode3::PushOnly,
// Kind3::Vec,
// )?,
// addresstype_to_addressindex_and_unspentoutpoint: Store::import(
// database_ref,
// path,
@@ -112,14 +159,29 @@ impl Stores {
pub fn commit(&mut self, height: Height) -> Result<()> {
[
&mut self.addresshash_to_typeindex as &mut dyn AnyStore,
&mut self.blockhashprefix_to_height,
&mut self.blockhashprefix_to_height as &mut dyn AnyStore,
&mut self.height_to_coinbase_tag,
&mut self.txidprefix_to_txindex,
&mut self.addresstype_to_addressindex_and_txindex,
// &mut self.addresshash_to_typeindex
// &mut self.addresstype_to_addressindex_and_txindex,
// &mut self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_par_iter() // Changed from par_iter_mut()
.into_par_iter()
.chain(
self.addresstype_to_addresshash_to_addressindex
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_txindex
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_unspentoutpoint
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
) // Changed from par_iter_mut()
.try_for_each(|store| store.commit(height))?;
self.database
@@ -129,14 +191,29 @@ impl Stores {
fn iter_any_store(&self) -> impl Iterator<Item = &dyn AnyStore> {
[
&self.addresshash_to_typeindex as &dyn AnyStore,
&self.blockhashprefix_to_height,
&self.blockhashprefix_to_height as &dyn AnyStore,
&self.height_to_coinbase_tag,
&self.txidprefix_to_txindex,
&self.addresstype_to_addressindex_and_txindex,
// &self.addresshash_to_typeindex,
// &self.addresstype_to_addressindex_and_txindex,
// &self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_iter()
.chain(
self.addresstype_to_addresshash_to_addressindex
.iter()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_txindex
.iter()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_unspentoutpoint
.iter()
.map(|s| s as &dyn AnyStore),
)
}
pub fn rollback_if_needed(
@@ -144,14 +221,26 @@ impl Stores {
vecs: &mut Vecs,
starting_indexes: &Indexes,
) -> Result<()> {
if self.addresshash_to_typeindex.is_empty()?
&& self.blockhashprefix_to_height.is_empty()?
if self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?
&& self.height_to_coinbase_tag.is_empty()?
&& self.addresstype_to_addressindex_and_txindex.is_empty()?
// && self
// .addresstype_to_addressindex_and_unspentoutpoint
// .is_empty()?
// && self.addresshash_to_typeindex.is_empty()?
// && self.addresstype_to_addressindex_and_txindex.is_empty()?
// && self
// .addresstype_to_addressindex_and_unspentoutpoint
// .is_empty()?
&& self
.addresstype_to_addresshash_to_addressindex
.iter()
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
&& self
.addresstype_to_addressindex_and_txindex
.iter()
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
&& self
.addresstype_to_addressindex_and_unspentoutpoint
.iter()
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
{
return Ok(());
}
@@ -173,7 +262,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2pk65addressindex
.read(starting_indexes.height)
.read_once(starting_indexes.height)
{
let mut p2pk65addressindex_to_p2pk65bytes_iter =
vecs.p2pk65addressindex_to_p2pk65bytes.iter()?;
@@ -181,14 +270,16 @@ impl Stores {
while let Some(typedbytes) = p2pk65addressindex_to_p2pk65bytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2PK65)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2pk33addressindex
.read(starting_indexes.height)
.read_once(starting_indexes.height)
{
let mut p2pk33addressindex_to_p2pk33bytes_iter =
vecs.p2pk33addressindex_to_p2pk33bytes.iter()?;
@@ -196,14 +287,16 @@ impl Stores {
while let Some(typedbytes) = p2pk33addressindex_to_p2pk33bytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2PK33)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2pkhaddressindex
.read(starting_indexes.height)
.read_once(starting_indexes.height)
{
let mut p2pkhaddressindex_to_p2pkhbytes_iter =
vecs.p2pkhaddressindex_to_p2pkhbytes.iter()?;
@@ -211,14 +304,16 @@ impl Stores {
while let Some(typedbytes) = p2pkhaddressindex_to_p2pkhbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2PKH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2shaddressindex
.read(starting_indexes.height)
.read_once(starting_indexes.height)
{
let mut p2shaddressindex_to_p2shbytes_iter =
vecs.p2shaddressindex_to_p2shbytes.iter()?;
@@ -226,29 +321,16 @@ impl Stores {
while let Some(typedbytes) = p2shaddressindex_to_p2shbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2traddressindex
.read(starting_indexes.height)
{
let mut p2traddressindex_to_p2trbytes_iter =
vecs.p2traddressindex_to_p2trbytes.iter()?;
while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2SH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2wpkhaddressindex
.read(starting_indexes.height)
.read_once(starting_indexes.height)
{
let mut p2wpkhaddressindex_to_p2wpkhbytes_iter =
vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter()?;
@@ -256,14 +338,16 @@ impl Stores {
while let Some(typedbytes) = p2wpkhaddressindex_to_p2wpkhbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2WPKH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2wshaddressindex
.read(starting_indexes.height)
.read_once(starting_indexes.height)
{
let mut p2wshaddressindex_to_p2wshbytes_iter =
vecs.p2wshaddressindex_to_p2wshbytes.iter()?;
@@ -271,14 +355,33 @@ impl Stores {
while let Some(typedbytes) = p2wshaddressindex_to_p2wshbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2WSH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2traddressindex
.read_once(starting_indexes.height)
{
let mut p2traddressindex_to_p2trbytes_iter =
vecs.p2traddressindex_to_p2trbytes.iter()?;
while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2TR)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2aaddressindex
.read(starting_indexes.height)
.read_once(starting_indexes.height)
{
let mut p2aaddressindex_to_p2abytes_iter =
vecs.p2aaddressindex_to_p2abytes.iter()?;
@@ -286,7 +389,9 @@ impl Stores {
while let Some(typedbytes) = p2aaddressindex_to_p2abytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresshash_to_typeindex.remove(hash);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2A)
.remove(hash);
index.increment();
}
}
@@ -339,9 +444,9 @@ impl Stores {
.for_each(|((txoutindex, addresstype), addressindex)| {
let txindex = txoutindex_to_txindex_iter.get_at_unwrap(txoutindex);
self.addresstype_to_addressindex_and_txindex.remove(
AddressTypeAddressIndexTxIndex::from((addresstype, addressindex, txindex)),
);
self.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.remove(AddressIndexTxIndex::from((addressindex, txindex)));
let vout = Vout::from(
txoutindex.to_usize()
@@ -351,13 +456,9 @@ impl Stores {
);
let outpoint = OutPoint::new(txindex, vout);
// self.addresstype_to_addressindex_and_unspentoutpoint.remove(
// AddressTypeAddressIndexOutPoint::from((
// addresstype,
// addressindex,
// outpoint,
// )),
// );
self.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.remove(AddressIndexOutPoint::from((addressindex, outpoint)));
});
// Add back outputs that were spent after the rollback point
@@ -386,22 +487,13 @@ impl Stores {
let addresstype = outputtype;
let addressindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex);
self.addresstype_to_addressindex_and_txindex.remove(
AddressTypeAddressIndexTxIndex::from((
addresstype,
addressindex,
txindex,
)),
);
self.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.remove(AddressIndexTxIndex::from((addressindex, txindex)));
// self.addresstype_to_addressindex_and_unspentoutpoint.insert(
// AddressTypeAddressIndexOutPoint::from((
// addresstype,
// addressindex,
// outpoint,
// )),
// Unit,
// );
self.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert(AddressIndexOutPoint::from((addressindex, outpoint)), Unit);
}
}
});

2
crates/brk_indexer/src/vecs.rs
View File

@@ -192,7 +192,7 @@ impl Vecs {
};
this.db.retain_regions(
this.iter_any_collectable()
this.iter_any_writable()
.flat_map(|v| v.region_names())
.collect(),
)?;

4
crates/brk_logger/src/lib.rs
View File

@@ -23,8 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
});
Builder::from_env(Env::default().default_filter_or(
"info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
// "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
// "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
"debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
))
.format(move |buf, record| {
let date_time = Timestamp::now()

4
crates/brk_query/src/async.rs
View File

@@ -53,14 +53,14 @@ impl AsyncQuery {
// metric: &str,
// index: Index,
// // params: &Params,
// ) -> Result<Vec<(String, &&dyn AnyCollectableVec)>> {
// ) -> Result<Vec<(String, &&dyn AnyWritableVec)>> {
// let query = self.0.clone();
// spawn_blocking(move || query.search_metric_with_index(metric, index)).await?
// }
// pub async fn format(
// &self,
// metrics: Vec<(String, &&dyn AnyCollectableVec)>,
// metrics: Vec<(String, &&dyn AnyWritableVec)>,
// params: &ParamsOpt,
// ) -> Result<Output> {
// let query = self.0.clone();

36
crates/brk_query/src/lib.rs
View File

@@ -11,7 +11,7 @@ use brk_types::{
Address, AddressStats, Format, Height, Index, IndexInfo, Limit, Metric, MetricCount,
Transaction, TxidPath,
};
use vecdb::{AnyCollectableVec, AnyStoredVec};
use vecdb::{AnyStoredVec, AnyWritableVec};
mod r#async;
mod chain;
@@ -77,7 +77,7 @@ impl Query {
metric: &str,
index: Index,
// params: &Params,
) -> Result<Vec<(String, &&dyn AnyCollectableVec)>> {
) -> Result<Vec<(String, &&dyn AnyWritableVec)>> {
todo!();
// let all_metrics = &self.vecs.metrics;
@@ -121,7 +121,7 @@ impl Query {
}
fn columns_to_csv(
columns: &[&&dyn AnyCollectableVec],
columns: &[&&dyn AnyWritableVec],
from: Option<i64>,
to: Option<i64>,
) -> Result<String> {
@@ -144,23 +144,31 @@ impl Query {
}
csv.push('\n');
let mut iters: Vec<_> = columns
// Create one writer per column
let mut writers: Vec<_> = columns
.iter()
.map(|col| col.iter_range_strings(from, to))
.map(|col| col.create_writer(from, to))
.collect();
for _ in 0..num_rows {
for (index, iter) in iters.iter_mut().enumerate() {
for (index, writer) in writers.iter_mut().enumerate() {
if index > 0 {
csv.push(',');
}
if let Some(field) = iter.next() {
if field.contains(',') {
// Check if we need CSV escaping
let start_pos = csv.len();
if writer.write_next(&mut csv)? {
let end_pos = csv.len();
// If contains comma, rewrite with quotes
if csv[start_pos..end_pos].contains(',') {
let value = csv[start_pos..end_pos].to_string(); // Only allocate if needed
csv.truncate(start_pos);
csv.push('"');
csv.push_str(&field);
csv.push_str(&value);
csv.push('"');
} else {
csv.push_str(&field);
}
}
}
@@ -170,11 +178,7 @@ impl Query {
Ok(csv)
}
pub fn format(
&self,
metrics: Vec<&&dyn AnyCollectableVec>,
params: &ParamsOpt,
) -> Result<Output> {
pub fn format(&self, metrics: Vec<&&dyn AnyWritableVec>, params: &ParamsOpt) -> Result<Output> {
let from = params.from().map(|from| {
metrics
.iter()

12
crates/brk_query/src/vecs.rs
View File

@@ -6,7 +6,7 @@ use brk_traversable::{Traversable, TreeNode};
use brk_types::{Index, IndexInfo, Limit, Metric};
use derive_deref::{Deref, DerefMut};
use quickmatch::{QuickMatch, QuickMatchConfig};
use vecdb::AnyCollectableVec;
use vecdb::AnyWritableVec;
use crate::pagination::{PaginatedIndexParam, PaginatedMetrics, PaginationParam};
@@ -31,11 +31,11 @@ impl<'a> Vecs<'a> {
indexer
.vecs
.iter_any_collectable()
.iter_any_writable()
.for_each(|vec| this.insert(vec));
computer
.iter_any_collectable()
.iter_any_writable()
.for_each(|vec| this.insert(vec));
let mut ids = this
@@ -108,7 +108,7 @@ impl<'a> Vecs<'a> {
}
// Not the most performant or type safe but only built once so that's okay
fn insert(&mut self, vec: &'a dyn AnyCollectableVec) {
fn insert(&mut self, vec: &'a dyn AnyWritableVec) {
let name = vec.name();
let serialized_index = vec.index_type_to_string();
let index = Index::try_from(serialized_index)
@@ -181,7 +181,7 @@ impl<'a> Vecs<'a> {
}
#[derive(Default, Deref, DerefMut)]
pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyCollectableVec>);
pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyWritableVec>);
#[derive(Default, Deref, DerefMut)]
pub struct MetricToVec<'a>(BTreeMap<&'a str, &'a dyn AnyCollectableVec>);
pub struct MetricToVec<'a>(BTreeMap<&'a str, &'a dyn AnyWritableVec>);

5
crates/brk_rpc/src/lib.rs
View File

@@ -17,7 +17,7 @@ pub use bitcoincore_rpc::Auth;
mod inner;
use inner::ClientInner;
use log::info;
use log::{debug, info};
///
/// Bitcoin Core RPC Client
@@ -67,6 +67,7 @@ impl Client {
/// Returns the numbers of block in the longest chain.
pub fn get_last_height(&self) -> Result<Height> {
debug!("Get last height...");
self.call(|c| c.get_block_count())
.map(Height::from)
.map_err(Into::into)
@@ -215,6 +216,8 @@ impl Client {
}
pub fn get_closest_valid_height(&self, hash: BlockHash) -> Result<(Height, BlockHash)> {
debug!("Get closest valid height...");
match self.get_block_header_info(&hash) {
Ok(block_info) => {
if self.is_in_main_chain(&hash)? {

7
crates/brk_store/src/any.rs
View File

@@ -1,11 +1,16 @@
use brk_error::Result;
use brk_types::{Height, Version};
use fjall2::{InnerItem, PartitionHandle};
pub trait AnyStore: Send + Sync {
fn commit(&mut self, height: Height) -> Result<()>;
fn name(&self) -> &'static str;
fn height(&self) -> Option<Height>;
fn has(&self, height: Height) -> bool;
fn needs(&self, height: Height) -> bool;
fn version(&self) -> Version;
fn export_meta_if_needed(&mut self, height: Height) -> Result<()>;
fn partition(&self) -> &PartitionHandle;
fn take_all_f2(&mut self) -> Vec<InnerItem>;
fn commit(&mut self) -> Result<()>;
// fn take_all_f3(&mut self) -> Box<dyn Iterator<Item = Item>>;
}

40
crates/brk_store/src/fjall_v2/mod.rs
View File

@@ -169,18 +169,42 @@ where
impl<K, V> AnyStore for StoreFjallV2<K, V>
where
K: Debug + Clone + From<ByteView> + Ord + Eq + Hash,
V: Debug + Clone + From<ByteView>,
K: Debug + Clone + From<ByteView> + Ord + Eq + Hash + 'static,
V: Debug + Clone + From<ByteView> + 'static,
ByteView: From<K> + From<V>,
Self: Send + Sync,
{
fn commit(&mut self, height: Height) -> Result<()> {
fn partition(&self) -> &fjall2::PartitionHandle {
self.partition.inner()
}
fn take_all_f2(&mut self) -> Vec<InnerItem> {
let mut items = mem::take(&mut self.puts)
.into_iter()
.map(|(key, value)| Item::Value { key, value })
.chain(
mem::take(&mut self.dels)
.into_iter()
.map(|key| Item::Tomb(key)),
)
.collect::<Vec<_>>();
items.sort_unstable();
items.into_iter().map(InnerItem::from).collect()
}
// fn take_all_f3(&mut self) -> Box<dyn Iterator<Item = Item>> {
// Box::new([].into_iter())
// }
fn export_meta_if_needed(&mut self, height: Height) -> Result<()> {
if self.has(height) {
return Ok(());
}
self.meta.export(height)?;
Ok(())
}
fn commit(&mut self) -> Result<()> {
if self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
}
@@ -208,10 +232,10 @@ where
.collect::<Vec<_>>();
items.sort_unstable();
self.keyspace.inner().batch().commit_partition(
self.partition.inner(),
items.into_iter().map(InnerItem::from).collect::<Vec<_>>(),
)?;
// self.keyspace.inner().batch().commit_partition(
// self.partition.inner(),
// items.into_iter().map(InnerItem::from).collect::<Vec<_>>(),
// )?;
// }
Ok(())

98
crates/brk_store/src/fjall_v3/mod.rs
View File

@@ -26,15 +26,13 @@ pub struct StoreFjallV3<Key, Value> {
keyspace: Keyspace,
puts: FxHashMap<Key, Value>,
dels: FxHashSet<Key>,
mode: Mode3,
kind: Kind3,
}
const MAJOR_FJALL_VERSION: Version = Version::new(3);
pub fn open_fjall3_database(path: &Path) -> fjall3::Result<Database> {
Database::builder(path.join("fjall"))
.cache_size(1024 * 1024 * 1024)
.cache_size(4 * 1024 * 1024 * 1024)
.open()
}
@@ -73,8 +71,6 @@ where
keyspace,
puts: FxHashMap::default(),
dels: FxHashSet::default(),
mode,
kind,
})
}
@@ -94,7 +90,9 @@ where
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(7.0)),
]));
} else {
options = options.filter_policy(FilterPolicy::disabled());
options = options
.max_memtable_size(8 * 1024 * 1024)
.filter_policy(FilterPolicy::disabled());
}
if kind.is_sequential() {
@@ -156,6 +154,14 @@ where
}
}
#[inline]
pub fn iter(&self) -> impl Iterator<Item = (K, V)> {
self.keyspace
.iter()
.map(|res| res.into_inner().unwrap())
.map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v))))
}
#[inline]
fn has(&self, height: Height) -> bool {
self.meta.has(height)
@@ -174,49 +180,63 @@ where
ByteView: From<K> + From<V>,
Self: Send + Sync,
{
fn commit(&mut self, height: Height) -> Result<()> {
fn take_all_f2(&mut self) -> Vec<fjall2::InnerItem> {
vec![]
}
fn partition(&self) -> &fjall2::PartitionHandle {
panic!()
}
// fn take_all_f3(&mut self) -> Box<dyn Iterator<Item = Item>> {
// Box::new([].into_iter())
// }
fn export_meta_if_needed(&mut self, height: Height) -> Result<()> {
if self.has(height) {
return Ok(());
}
self.meta.export(height)?;
Ok(())
}
fn commit(&mut self) -> Result<()> {
if self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
}
if self.mode.is_push_only() {
if !self.dels.is_empty() {
unreachable!();
// if self.mode.is_push_only() {
// if !self.dels.is_empty() {
// unreachable!();
// }
// let mut puts = mem::take(&mut self.puts).into_iter().collect::<Vec<_>>();
// puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2));
// // dbg!(&puts);
// self.keyspace.ingest(
// puts.into_iter()
// .map(|(k, v)| (ByteView::from(k), ByteView::from(v))),
// )?;
// } else {
let mut batch = self.database.batch();
// let mut batch = self.database.inner().batch();
let mut items = mem::take(&mut self.puts)
.into_iter()
.map(|(key, value)| Item::Value { key, value })
.chain(
mem::take(&mut self.dels)
.into_iter()
.map(|key| Item::Tomb(key)),
)
.collect::<Vec<_>>();
items.sort_unstable();
items.into_iter().for_each(|item| match item {
Item::Value { key, value } => {
batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value))
}
let mut puts = mem::take(&mut self.puts).into_iter().collect::<Vec<_>>();
puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2));
// dbg!(&puts);
self.keyspace.ingest(
puts.into_iter()
.map(|(k, v)| (ByteView::from(k), ByteView::from(v))),
)?;
} else {
let mut batch = self.database.batch();
// let mut batch = self.database.inner().batch();
let mut items = mem::take(&mut self.puts)
.into_iter()
.map(|(key, value)| Item::Value { key, value })
.chain(
mem::take(&mut self.dels)
.into_iter()
.map(|key| Item::Tomb(key)),
)
.collect::<Vec<_>>();
items.sort_unstable();
items.into_iter().for_each(|item| match item {
Item::Value { key, value } => {
batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value))
}
Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)),
});
batch.commit()?;
}
Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)),
});
batch.commit()?;
// }
// batch.ingest(
// items

54
crates/brk_traversable/src/lib.rs
View File

@@ -5,13 +5,13 @@ pub use brk_types::TreeNode;
#[cfg(feature = "derive")]
pub use brk_traversable_derive::Traversable;
use vecdb::{
AnyCollectableVec, AnyVec, CompressedVec, ComputedVec, EagerVec, LazyVecFrom1, LazyVecFrom2,
AnyVec, AnyWritableVec, CompressedVec, ComputedVec, EagerVec, LazyVecFrom1, LazyVecFrom2,
LazyVecFrom3, RawVec, StoredCompressed, StoredIndex, StoredRaw, StoredVec,
};
pub trait Traversable {
fn to_tree_node(&self) -> TreeNode;
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec>;
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec>;
}
impl<I, T> Traversable for RawVec<I, T>
@@ -19,8 +19,8 @@ where
I: StoredIndex,
T: StoredRaw,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -33,8 +33,8 @@ where
I: StoredIndex,
T: StoredCompressed,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -47,8 +47,8 @@ where
I: StoredIndex,
T: StoredCompressed,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -61,8 +61,8 @@ where
I: StoredIndex,
T: StoredCompressed,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -77,8 +77,8 @@ where
S1I: StoredIndex,
S1T: StoredRaw,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -95,8 +95,8 @@ where
S2I: StoredIndex,
S2T: StoredRaw,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -116,8 +116,8 @@ where
S3I: StoredIndex,
S3T: StoredRaw,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -137,8 +137,8 @@ where
S3I: StoredIndex,
S3T: StoredCompressed,
{
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
std::iter::once(self as &dyn AnyCollectableVec)
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
std::iter::once(self as &dyn AnyWritableVec)
}
fn to_tree_node(&self) -> TreeNode {
@@ -151,8 +151,8 @@ impl<T: Traversable + ?Sized> Traversable for Box<T> {
(**self).to_tree_node()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
(**self).iter_any_collectable()
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
(**self).iter_any_writable()
}
}
@@ -164,10 +164,11 @@ impl<T: Traversable> Traversable for Option<T> {
}
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
match self {
Some(inner) => Box::new(inner.iter_any_collectable())
as Box<dyn Iterator<Item = &dyn AnyCollectableVec>>,
Some(inner) => {
Box::new(inner.iter_any_writable()) as Box<dyn Iterator<Item = &dyn AnyWritableVec>>
}
None => Box::new(std::iter::empty()),
}
}
@@ -182,11 +183,10 @@ impl<K: Debug, V: Traversable> Traversable for BTreeMap<K, V> {
TreeNode::Branch(children)
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(std::iter::empty());
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
let mut iter: Box<dyn Iterator<Item = &dyn AnyWritableVec>> = Box::new(std::iter::empty());
for v in self.values() {
iter = Box::new(iter.chain(v.iter_any_collectable()));
iter = Box::new(iter.chain(v.iter_any_writable()));
}
iter
}

20
crates/brk_traversable_derive/src/lib.rs
View File

@@ -29,8 +29,8 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream {
self.0.to_tree_node()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
self.0.iter_any_collectable()
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn vecdb::AnyWritableVec> {
self.0.iter_any_writable()
}
}
});
@@ -44,7 +44,7 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream {
brk_traversable::TreeNode::Branch(std::collections::BTreeMap::new())
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn vecdb::AnyWritableVec> {
std::iter::empty()
}
}
@@ -281,7 +281,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream {
if regular_fields.is_empty() && option_fields.is_empty() {
return quote! {
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn vecdb::AnyWritableVec> {
std::iter::empty()
}
};
@@ -290,17 +290,17 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream {
let (init_part, chain_part) = if let Some((&first, rest)) = regular_fields.split_first() {
(
quote! {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(self.#first.iter_any_collectable());
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyWritableVec>> =
Box::new(self.#first.iter_any_writable());
},
quote! {
#(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_collectable()));)*
#(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_writable()));)*
},
)
} else {
(
quote! {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyWritableVec>> =
Box::new(std::iter::empty());
},
quote! {},
@@ -311,7 +311,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream {
let chains = option_fields.iter().map(|f| {
quote! {
if let Some(ref x) = self.#f {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(x.iter_any_writable()));
}
}
});
@@ -321,7 +321,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream {
};
quote! {
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn vecdb::AnyWritableVec> {
#init_part
#chain_part
#option_part