Compare commits

..

56 Commits

Author SHA1 Message Date
nym21 be9569f3fb release: v0.0.82 2025-07-26 22:09:21 +02:00
nym21 900e72f95a cargo: cleanup deps 2025-07-26 14:28:26 +02:00
nym21 d2827f188b computer: temp remove rayon 2025-07-26 14:24:06 +02:00
nym21 cf9903b759 computer: init file with min length and regions 2025-07-26 08:57:13 +02:00
nym21 23f96461f4 computer: remove libc dep 2025-07-26 08:42:33 +02:00
nym21 9f2fd26e98 computer: fixes 2025-07-26 08:41:19 +02:00
nym21 78d837c080 computer: flush + punch 2025-07-26 01:04:36 +02:00
nym21 241b9312b7 cli: config changes 2025-07-26 00:46:35 +02:00
nym21 ed70ad7378 indexer: take readers before last export 2025-07-25 22:45:41 +02:00
nym21 00213176d8 indexer: small changes 2025-07-25 22:38:15 +02:00
nym21 406650a45a vec: removed 2025-07-25 20:38:57 +02:00
nym21 56750ccf3c vecs: part 11 2025-07-25 20:27:15 +02:00
nym21 dfc286b393 vecs: part 10 2025-07-25 20:22:54 +02:00
nym21 49a66f72fc crates: update rapidhash 2025-07-24 17:32:38 +02:00
nym21 3f237689da vecs: part 9 2025-07-24 17:19:05 +02:00
nym21 cf1fb483b3 vecs: part 8 2025-07-24 16:48:50 +02:00
nym21 b10f5e3f67 vecs: part 7 2025-07-23 23:55:13 +02:00
nym21 c4fc24c513 vecs: part 6 2025-07-23 09:17:26 +02:00
nym21 3ac9c2d95e vecs: part 5 2025-07-22 21:26:50 +02:00
nym21 e5ab4dafc0 vecs: part 4 2025-07-22 17:36:34 +02:00
nym21 10ae1911c3 vecs: part 3 2025-07-22 15:10:07 +02:00
nym21 73ebcdf0d6 vecs: part 2 2025-07-22 13:19:19 +02:00
nym21 5347523921 vecs: init 2025-07-21 11:02:25 +02:00
nym21 7ef70b953b vec: lazy: remove unneeded phantoms 2025-07-19 17:47:25 +02:00
nym21 ccaca524fe computer: libc sync 2025-07-19 10:10:01 +02:00
nym21 dd51f91cab computer: final fix for external disks crashing 2025-07-18 16:29:53 +02:00
nym21 537d98b41b release: v0.0.81 2025-07-17 23:45:01 +02:00
nym21 9c4cadfc04 vec: fix holes export 2025-07-17 17:29:53 +02:00
nym21 2001370441 mcp: use rust-rmcp instead of brk-rmcp 2025-07-17 16:34:29 +02:00
nym21 cc87b22757 computer: perf improvements 2025-07-17 16:17:21 +02:00
nym21 c0a65b30ad indexer: update example 2025-07-17 11:39:25 +02:00
nym21 c07e66c086 computer: fix stateful 2025-07-17 11:35:40 +02:00
nym21 a0cfc1be2b computer: convert stores to vecs part 2 2025-07-16 16:23:40 +02:00
nym21 1505454793 computer: convert stores to vecs part 1 2025-07-15 22:47:46 +02:00
nym21 e1dff66283 pr: merge #21 from deadmanoz/dockerize
Add Docker support
2025-07-15 15:51:32 +00:00
deadmanoz 5be801a086 Merge branch 'main' into dockerize 2025-07-15 08:50:22 -07:00
deadmanoz 94d4b05c29 Address review feedback 2025-07-15 08:48:39 -07:00
nym21 cebb889f7e cargo: update 2025-07-14 16:00:31 +02:00
nym21 c4ed6ed034 store: remove rotate_memtable as could be the root cause of the issue 2025-07-14 15:48:19 +02:00
nym21 ec960bfefa release: v0.0.80 2025-07-13 21:20:40 +02:00
nym21 79f689dde1 mcp: remove claude results examples due to dead links 2025-07-13 21:20:02 +02:00
nym21 3b3654df56 vec: add local and shared stored_len to raw variant 2025-07-13 19:30:50 +02:00
nym21 c66f008f07 release: v0.0.79 2025-07-13 17:18:14 +02:00
nym21 37d9498d90 crates: upgrade 2025-07-13 17:18:02 +02:00
nym21 1ff67093db website: apply datasets changes to charts 2025-07-13 17:14:34 +02:00
nym21 daed37ccb8 stores: forgot some files 2025-07-13 16:52:19 +02:00
nym21 d41d807b4f stores: bloom filters back to default due to slow reads, v3 will bring down the needed RAM 2025-07-13 16:49:45 +02:00
nym21 d6fa5c8a55 vec: fix header reading of existing file 2025-07-13 16:31:22 +02:00
nym21 2dd608dfed vec: don't store mmap in struct anymore 2025-07-13 11:50:34 +02:00
deadmanoz 5de9757d46 Remove services from docker 2025-07-04 16:37:39 +08:00
deadmanoz f89276d7b8 Remove redundant services 2025-07-04 15:51:28 +08:00
deadmanoz 30ba034206 Move docker artefacts into /docker directory 2025-07-04 13:00:12 +08:00
deadmanoz fa1e5aaa7f Make Parser::new the only entrypoint 2025-07-04 12:15:32 +08:00
deadmanoz 870c70180f Back to a single image/container setup 2025-07-04 11:40:37 +08:00
deadmanoz d83a833b4d Switch to multiple container setup 2025-06-27 12:56:25 +08:00
deadmanoz ec3a2f29f0 Docker functionality, change location of 'blk_index_to_blk_recap.json' 2025-06-27 12:56:03 +08:00
167 changed files with 6893 additions and 4421 deletions
+6
View File
@@ -19,7 +19,13 @@ _*
# Logs # Logs
.log .log
# Environment variables/configs
.env
# Profiling # Profiling
profile.json.gz profile.json.gz
flamegraph.svg flamegraph.svg
*.trace *.trace
# AI
CLAUDE.md
Generated
+166 -127
View File
@@ -144,12 +144,6 @@ dependencies = [
"derive_arbitrary", "derive_arbitrary",
] ]
[[package]]
name = "arc-swap"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
[[package]] [[package]]
name = "arcstr" name = "arcstr"
version = "1.2.0" version = "1.2.0"
@@ -170,9 +164,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]] [[package]]
name = "async-compression" name = "async-compression"
version = "0.4.25" version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4" checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8"
dependencies = [ dependencies = [
"brotli", "brotli",
"flate2", "flate2",
@@ -477,7 +471,7 @@ dependencies = [
[[package]] [[package]]
name = "brk" name = "brk"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"brk_bundler", "brk_bundler",
"brk_cli", "brk_cli",
@@ -492,12 +486,12 @@ dependencies = [
"brk_parser", "brk_parser",
"brk_server", "brk_server",
"brk_store", "brk_store",
"brk_vec", "brk_vecs",
] ]
[[package]] [[package]]
name = "brk_bundler" name = "brk_bundler"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"brk_rolldown", "brk_rolldown",
"log", "log",
@@ -508,7 +502,7 @@ dependencies = [
[[package]] [[package]]
name = "brk_cli" name = "brk_cli"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"bitcoincore-rpc", "bitcoincore-rpc",
"brk_computer", "brk_computer",
@@ -519,7 +513,7 @@ dependencies = [
"brk_logger", "brk_logger",
"brk_parser", "brk_parser",
"brk_server", "brk_server",
"brk_vec", "brk_vecs",
"clap", "clap",
"clap_derive", "clap_derive",
"color-eyre", "color-eyre",
@@ -531,7 +525,7 @@ dependencies = [
[[package]] [[package]]
name = "brk_computer" name = "brk_computer"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"bincode", "bincode",
"bitcoin", "bitcoin",
@@ -542,13 +536,9 @@ dependencies = [
"brk_indexer", "brk_indexer",
"brk_logger", "brk_logger",
"brk_parser", "brk_parser",
"brk_store", "brk_vecs",
"brk_vec",
"color-eyre", "color-eyre",
"derive_deref", "derive_deref",
"either",
"fjall",
"jiff",
"log", "log",
"rayon", "rayon",
"serde", "serde",
@@ -558,7 +548,7 @@ dependencies = [
[[package]] [[package]]
name = "brk_core" name = "brk_core"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"bincode", "bincode",
"bitcoin", "bitcoin",
@@ -579,16 +569,17 @@ dependencies = [
[[package]] [[package]]
name = "brk_exit" name = "brk_exit"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"brk_logger", "brk_logger",
"ctrlc", "ctrlc",
"log", "log",
"parking_lot",
] ]
[[package]] [[package]]
name = "brk_fetcher" name = "brk_fetcher"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"brk_core", "brk_core",
"brk_logger", "brk_logger",
@@ -600,7 +591,7 @@ dependencies = [
[[package]] [[package]]
name = "brk_indexer" name = "brk_indexer"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"bitcoin", "bitcoin",
"bitcoincore-rpc", "bitcoincore-rpc",
@@ -609,7 +600,7 @@ dependencies = [
"brk_logger", "brk_logger",
"brk_parser", "brk_parser",
"brk_store", "brk_store",
"brk_vec", "brk_vecs",
"color-eyre", "color-eyre",
"fjall", "fjall",
"log", "log",
@@ -618,15 +609,15 @@ dependencies = [
[[package]] [[package]]
name = "brk_interface" name = "brk_interface"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"brk_computer", "brk_computer",
"brk_core", "brk_core",
"brk_indexer", "brk_indexer",
"brk_rmcp", "brk_vecs",
"brk_vec",
"color-eyre", "color-eyre",
"derive_deref", "derive_deref",
"rmcp",
"schemars 1.0.4", "schemars 1.0.4",
"serde", "serde",
"serde_json", "serde_json",
@@ -636,7 +627,7 @@ dependencies = [
[[package]] [[package]]
name = "brk_logger" name = "brk_logger"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"color-eyre", "color-eyre",
"env_logger", "env_logger",
@@ -646,17 +637,17 @@ dependencies = [
[[package]] [[package]]
name = "brk_mcp" name = "brk_mcp"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"axum", "axum",
"brk_interface", "brk_interface",
"brk_rmcp",
"log", "log",
"rmcp",
] ]
[[package]] [[package]]
name = "brk_parser" name = "brk_parser"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"bitcoin", "bitcoin",
"bitcoincore-rpc", "bitcoincore-rpc",
@@ -669,49 +660,6 @@ dependencies = [
"zerocopy", "zerocopy",
] ]
[[package]]
name = "brk_rmcp"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dde96460b07f97f8522cef62fa5e9d5271a5183b13e6d75379ab43c1fb0b106"
dependencies = [
"base64 0.22.1",
"brk_rmcp-macros",
"bytes",
"chrono",
"futures",
"http",
"http-body",
"http-body-util",
"paste",
"pin-project-lite",
"rand 0.9.1",
"schemars 1.0.4",
"serde",
"serde_json",
"sse-stream",
"thiserror 2.0.12",
"tokio",
"tokio-stream",
"tokio-util",
"tower-service",
"tracing",
"uuid",
]
[[package]]
name = "brk_rmcp-macros"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48a3cb1aadff2f3aaaed76d17e71d6b828cafbd3fe9fbaba07d31b8ed10e63c2"
dependencies = [
"darling",
"proc-macro2",
"quote",
"serde_json",
"syn 2.0.104",
]
[[package]] [[package]]
name = "brk_rolldown" name = "brk_rolldown"
version = "0.1.1" version = "0.1.1"
@@ -1025,7 +973,7 @@ dependencies = [
[[package]] [[package]]
name = "brk_server" name = "brk_server"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"axum", "axum",
"bitcoincore-rpc", "bitcoincore-rpc",
@@ -1039,7 +987,7 @@ dependencies = [
"brk_logger", "brk_logger",
"brk_mcp", "brk_mcp",
"brk_parser", "brk_parser",
"brk_vec", "brk_vecs",
"clap", "clap",
"clap_derive", "clap_derive",
"color-eyre", "color-eyre",
@@ -1047,6 +995,7 @@ dependencies = [
"log", "log",
"minreq", "minreq",
"serde", "serde",
"serde_json",
"tokio", "tokio",
"tower-http", "tower-http",
"tracing", "tracing",
@@ -1055,7 +1004,7 @@ dependencies = [
[[package]] [[package]]
name = "brk_store" name = "brk_store"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"brk_core", "brk_core",
"byteview", "byteview",
@@ -1076,16 +1025,17 @@ dependencies = [
] ]
[[package]] [[package]]
name = "brk_vec" name = "brk_vecs"
version = "0.0.78" version = "0.0.82"
dependencies = [ dependencies = [
"arc-swap",
"brk_core", "brk_core",
"brk_exit", "brk_exit",
"clap", "clap",
"clap_derive", "clap_derive",
"libc",
"log", "log",
"memmap2", "memmap2",
"parking_lot",
"rayon", "rayon",
"serde", "serde",
"serde_json", "serde_json",
@@ -1159,18 +1109,18 @@ dependencies = [
[[package]] [[package]]
name = "castaway" name = "castaway"
version = "0.2.3" version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
dependencies = [ dependencies = [
"rustversion", "rustversion",
] ]
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.29" version = "1.2.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
dependencies = [ dependencies = [
"jobserver", "jobserver",
"libc", "libc",
@@ -1371,9 +1321,9 @@ dependencies = [
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.4.2" version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
] ]
@@ -1479,8 +1429,18 @@ version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
dependencies = [ dependencies = [
"darling_core", "darling_core 0.20.11",
"darling_macro", "darling_macro 0.20.11",
]
[[package]]
name = "darling"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a79c4acb1fd5fa3d9304be4c76e031c54d2e92d172a393e24b19a14fe8532fe9"
dependencies = [
"darling_core 0.21.0",
"darling_macro 0.21.0",
] ]
[[package]] [[package]]
@@ -1497,13 +1457,38 @@ dependencies = [
"syn 2.0.104", "syn 2.0.104",
] ]
[[package]]
name = "darling_core"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74875de90daf30eb59609910b84d4d368103aaec4c924824c6799b28f77d6a1d"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 2.0.104",
]
[[package]] [[package]]
name = "darling_macro" name = "darling_macro"
version = "0.20.11" version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [ dependencies = [
"darling_core", "darling_core 0.20.11",
"quote",
"syn 2.0.104",
]
[[package]]
name = "darling_macro"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e79f8e61677d5df9167cd85265f8e5f64b215cdea3fb55eebc3e622e44c7a146"
dependencies = [
"darling_core 0.21.0",
"quote", "quote",
"syn 2.0.104", "syn 2.0.104",
] ]
@@ -1729,9 +1714,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]] [[package]]
name = "fjall" name = "fjall"
version = "2.11.1" version = "2.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5cb653019268f6dc8de3b254b633a2d233a775054349b804b9cfbf18bbe3426" checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"byteview", "byteview",
@@ -2045,9 +2030,9 @@ dependencies = [
[[package]] [[package]]
name = "hyper-util" name = "hyper-util"
version = "0.1.15" version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df" checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures-core", "futures-core",
@@ -2166,9 +2151,9 @@ dependencies = [
[[package]] [[package]]
name = "io-uring" name = "io-uring"
version = "0.7.8" version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
dependencies = [ dependencies = [
"bitflags 2.9.1", "bitflags 2.9.1",
"cfg-if", "cfg-if",
@@ -2338,9 +2323,9 @@ dependencies = [
[[package]] [[package]]
name = "libredox" name = "libredox"
version = "0.1.4" version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638" checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0"
dependencies = [ dependencies = [
"bitflags 2.9.1", "bitflags 2.9.1",
"libc", "libc",
@@ -2380,9 +2365,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]] [[package]]
name = "lsm-tree" name = "lsm-tree"
version = "2.10.1" version = "2.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2bd4cdc451a8dcf11329190afb9b78eb8988bed07a3da29b8d73d2e0c731ff" checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"crossbeam-skiplist", "crossbeam-skiplist",
@@ -2431,9 +2416,9 @@ checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
[[package]] [[package]]
name = "memmap2" name = "memmap2"
version = "0.9.5" version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28"
dependencies = [ dependencies = [
"libc", "libc",
] ]
@@ -3113,6 +3098,16 @@ dependencies = [
"unicode-width", "unicode-width",
] ]
[[package]]
name = "parking_lot"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]] [[package]]
name = "parking_lot_core" name = "parking_lot_core"
version = "0.9.11" version = "0.9.11"
@@ -3360,9 +3355,9 @@ dependencies = [
[[package]] [[package]]
name = "quick_cache" name = "quick_cache"
version = "0.6.14" version = "0.6.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b450dad8382b1b95061d5ca1eb792081fb082adf48c678791fe917509596d5f" checksum = "8565e62e02af316570d4b492f17af1481d6c07cea60f4e7edd71700da5052ba9"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.15.4", "hashbrown 0.15.4",
@@ -3406,9 +3401,9 @@ dependencies = [
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.9.1" version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [ dependencies = [
"rand_chacha 0.9.0", "rand_chacha 0.9.0",
"rand_core 0.9.3", "rand_core 0.9.3",
@@ -3454,9 +3449,9 @@ dependencies = [
[[package]] [[package]]
name = "rapidhash" name = "rapidhash"
version = "1.4.0" version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9813f789f95ee4fe6b4d01834404d7cccacbc3f6c029343af910b3c2835eb9f1" checksum = "2297bf4643b32dc85bf5f622c9a41a268db6c6a6a2a2e3b405958a5b014ad9e1"
[[package]] [[package]]
name = "rayon" name = "rayon"
@@ -3480,9 +3475,9 @@ dependencies = [
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.13" version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec"
dependencies = [ dependencies = [
"bitflags 2.9.1", "bitflags 2.9.1",
] ]
@@ -3553,9 +3548,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]] [[package]]
name = "regress" name = "regress"
version = "0.10.3" version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366" checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010"
dependencies = [ dependencies = [
"hashbrown 0.15.4", "hashbrown 0.15.4",
"memchr", "memchr",
@@ -3584,6 +3579,49 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "rmcp"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b008b927a85d514699ff304be84c5084598596e6cad4a6f5bc67207715fafe5f"
dependencies = [
"base64 0.22.1",
"bytes",
"chrono",
"futures",
"http",
"http-body",
"http-body-util",
"paste",
"pin-project-lite",
"rand 0.9.2",
"rmcp-macros",
"schemars 1.0.4",
"serde",
"serde_json",
"sse-stream",
"thiserror 2.0.12",
"tokio",
"tokio-stream",
"tokio-util",
"tower-service",
"tracing",
"uuid",
]
[[package]]
name = "rmcp-macros"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7465280d5f73f2c5c99017a04af407b2262455a149f255ad22f2b0b29087695c"
dependencies = [
"darling 0.21.0",
"proc-macro2",
"quote",
"serde_json",
"syn 2.0.104",
]
[[package]] [[package]]
name = "rolldown-ariadne" name = "rolldown-ariadne"
version = "0.5.2" version = "0.5.2"
@@ -3618,15 +3656,15 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "1.0.7" version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
dependencies = [ dependencies = [
"bitflags 2.9.1", "bitflags 2.9.1",
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
"windows-sys 0.59.0", "windows-sys 0.60.2",
] ]
[[package]] [[package]]
@@ -3696,6 +3734,7 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
dependencies = [ dependencies = [
"chrono",
"dyn-clone", "dyn-clone",
"ref-cast", "ref-cast",
"schemars_derive", "schemars_derive",
@@ -3816,9 +3855,9 @@ dependencies = [
[[package]] [[package]]
name = "serde_json" name = "serde_json"
version = "1.0.140" version = "1.0.141"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
dependencies = [ dependencies = [
"indexmap 2.10.0", "indexmap 2.10.0",
"itoa", "itoa",
@@ -3884,7 +3923,7 @@ version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f"
dependencies = [ dependencies = [
"darling", "darling 0.20.11",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.104", "syn 2.0.104",
@@ -4240,9 +4279,9 @@ dependencies = [
[[package]] [[package]]
name = "toml" name = "toml"
version = "0.9.1" version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0207d6ed1852c2a124c1fbec61621acb8330d2bf969a5d0643131e9affd985a5" checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac"
dependencies = [ dependencies = [
"indexmap 2.10.0", "indexmap 2.10.0",
"serde", "serde",
@@ -4264,18 +4303,18 @@ dependencies = [
[[package]] [[package]]
name = "toml_parser" name = "toml_parser"
version = "1.0.0" version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5c1c469eda89749d2230d8156a5969a69ffe0d6d01200581cdc6110674d293e" checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30"
dependencies = [ dependencies = [
"winnow", "winnow",
] ]
[[package]] [[package]]
name = "toml_writer" name = "toml_writer"
version = "1.0.0" version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b679217f2848de74cabd3e8fc5e6d66f40b7da40f8e1954d92054d9010690fd5" checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64"
[[package]] [[package]]
name = "tower" name = "tower"
@@ -4574,9 +4613,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]] [[package]]
name = "vfs" name = "vfs"
version = "0.12.1" version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ec343ec20aa715908fd028a4b8e7c99a349d13143224222e4d61c316d1e7f0a" checksum = "9e723b9e1c02a3cf9f9d0de6a4ddb8cdc1df859078902fe0ae0589d615711ae6"
dependencies = [ dependencies = [
"filetime", "filetime",
] ]
@@ -4929,9 +4968,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.7.11" version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"
[[package]] [[package]]
name = "wit-bindgen-rt" name = "wit-bindgen-rt"
+24 -20
View File
@@ -4,9 +4,10 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT" package.license = "MIT"
package.edition = "2024" package.edition = "2024"
package.version = "0.0.78" package.version = "0.0.82"
package.homepage = "https://bitcoinresearchkit.org" package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk" package.repository = "https://github.com/bitcoinresearchkit/brk"
package.readme = "README.md"
[profile.release] [profile.release]
lto = "fat" lto = "fat"
@@ -21,42 +22,45 @@ debug = true
inherits = "release" inherits = "release"
[workspace.dependencies] [workspace.dependencies]
arc-swap = "1.7.1"
axum = "0.8.4" axum = "0.8.4"
bincode = { version = "2.0.1", features = ["serde"] } bincode = { version = "2.0.1", features = ["serde"] }
bitcoin = { version = "0.32.6", features = ["serde"] } bitcoin = { version = "0.32.6", features = ["serde"] }
bitcoincore-rpc = "0.19.0" bitcoincore-rpc = "0.19.0"
brk_bundler = { version = "0.0.78", path = "crates/brk_bundler" } brk_bundler = { version = "0.0.82", path = "crates/brk_bundler" }
brk_cli = { version = "0.0.78", path = "crates/brk_cli" } brk_cli = { version = "0.0.82", path = "crates/brk_cli" }
brk_computer = { version = "0.0.78", path = "crates/brk_computer" } brk_computer = { version = "0.0.82", path = "crates/brk_computer" }
brk_core = { version = "0.0.78", path = "crates/brk_core" } brk_core = { version = "0.0.82", path = "crates/brk_core" }
brk_exit = { version = "0.0.78", path = "crates/brk_exit" } brk_exit = { version = "0.0.82", path = "crates/brk_exit" }
brk_fetcher = { version = "0.0.78", path = "crates/brk_fetcher" } brk_fetcher = { version = "0.0.82", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.0.78", path = "crates/brk_indexer" } brk_indexer = { version = "0.0.82", path = "crates/brk_indexer" }
brk_interface = { version = "0.0.78", path = "crates/brk_interface" } brk_interface = { version = "0.0.82", path = "crates/brk_interface" }
brk_logger = { version = "0.0.78", path = "crates/brk_logger" } brk_logger = { version = "0.0.82", path = "crates/brk_logger" }
brk_mcp = { version = "0.0.78", path = "crates/brk_mcp" } brk_mcp = { version = "0.0.82", path = "crates/brk_mcp" }
brk_parser = { version = "0.0.78", path = "crates/brk_parser" } brk_parser = { version = "0.0.82", path = "crates/brk_parser" }
brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]} brk_server = { version = "0.0.82", path = "crates/brk_server" }
# brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]} brk_store = { version = "0.0.82", path = "crates/brk_store" }
brk_server = { version = "0.0.78", path = "crates/brk_server" } brk_vecs = { version = "0.0.82", path = "crates/brk_vecs" }
brk_store = { version = "0.0.78", path = "crates/brk_store" }
brk_vec = { version = "0.0.78", path = "crates/brk_vec" }
byteview = "=0.6.1" byteview = "=0.6.1"
clap = { version = "4.5.41", features = ["string"] } clap = { version = "4.5.41", features = ["string"] }
clap_derive = "4.5.41" clap_derive = "4.5.41"
color-eyre = "0.6.5" color-eyre = "0.6.5"
derive_deref = "1.1.1" derive_deref = "1.1.1"
fjall = "2.11.1" fjall = "2.11.2"
jiff = "0.2.15" jiff = "0.2.15"
libc = "0.2.174"
log = { version = "0.4.27" } log = { version = "0.4.27" }
minreq = { version = "2.14.0", features = ["https", "serde_json"] } minreq = { version = "2.14.0", features = ["https", "serde_json"] }
parking_lot = "0.12.4"
rayon = "1.10.0" rayon = "1.10.0"
rmcp = { version = "0.3.0", features = [
"transport-worker",
"transport-streamable-http-server",
] }
schemars = "1.0.4" schemars = "1.0.4"
serde = { version = "1.0.219" } serde = { version = "1.0.219" }
serde_bytes = "0.11.17" serde_bytes = "0.11.17"
serde_derive = "1.0.219" serde_derive = "1.0.219"
serde_json = { version = "1.0.140", features = ["float_roundtrip"] } serde_json = { version = "1.0.141", features = ["float_roundtrip"] }
tabled = "0.20.0" tabled = "0.20.0"
tokio = { version = "1.46.1", features = ["rt-multi-thread"] } tokio = { version = "1.46.1", features = ["rt-multi-thread"] }
zerocopy = { version = "0.8.26" } zerocopy = { version = "0.8.26" }
+3 -4
View File
@@ -40,14 +40,13 @@ The toolkit can be used in various ways to accommodate as many needs as possible
- **[Website](https://bitcoinresearchkit.org)** \ - **[Website](https://bitcoinresearchkit.org)** \
Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \ Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \ It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz) Also available at: [brekit.org](https://brekit.org) // [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
- **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \ - **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \ Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \
Just like the website, it's entirely free, with no authentication or rate-limiting. Just like the website, it's entirely free, with no authentication or rate-limiting.
- **[AI](https://github.com/bitcoinresearchkit/brk/blob/main/crates/brk_mcp/README.md#brk-mcp)** \ - **[AI](https://github.com/bitcoinresearchkit/brk/blob/main/crates/brk_mcp/README.md#brk-mcp)** \
LLMs have to possibility to connect to BRK's backend through a [MCP](https://modelcontextprotocol.io/introduction). \ LLMs have to possibility to connect to BRK's backend through a [MCP](https://modelcontextprotocol.io/introduction). \
It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts. \ It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts.
One-shot output examples: [Document](https://claude.ai/public/artifacts/71194d29-f965-417c-ba09-fdf0e4ecb1d5) // [Dashboard](https://claude.ai/public/artifacts/beef143f-399a-4ed4-b8bf-c986b776de42) // [Dashboard 2](https://claude.ai/public/artifacts/5430ae49-bb3d-4fc1-ab24-f1e33deb40dc)
- **[CLI](https://crates.io/crates/brk_cli)** \ - **[CLI](https://crates.io/crates/brk_cli)** \
Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \ Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \
The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end. The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end.
@@ -74,7 +73,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine - [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer` - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec - [`brk_vec`](https://crates.io/crates/brk_vec): A storeable vec
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/) - [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
## Hosting as a service ## Hosting as a service
+6 -1
View File
@@ -10,8 +10,12 @@
- pull latest version and notify is out of date - pull latest version and notify is out of date
- _computer_ - _computer_
- **add rollback of states (in stateful)** - **add rollback of states (in stateful)**
- remove configurable format (raw/compressed) and chose sane ones instead
- linear reads: compressed (height/date/... + txindex_to_height + txindex_to_version + ...)
- random reads: raw (outputindex_to_value + ...)
- add prices paid by percentile (percentile cost basis) back
- add support for per index computation - add support for per index computation
- fix feerate which is always ZERO due to coinbase transaction - fix min feerate which is always ZERO due to coinbase transaction
- before computing multiple sources check their length, panic if not equal - before computing multiple sources check their length, panic if not equal
- add oracle price dataset (https://utxo.live/oracle/UTXOracle.py) - add oracle price dataset (https://utxo.live/oracle/UTXOracle.py)
- add address counts relative to all datasets - add address counts relative to all datasets
@@ -56,6 +60,7 @@
- add extensions support (.json .csv …) - add extensions support (.json .csv …)
- if format instead of extension then don't download file - if format instead of extension then don't download file
- add support for https (rustls) - add support for https (rustls)
- lru cache
- _vec_ - _vec_
- add native lock file support (once it's available in stable rust) - add native lock file support (once it's available in stable rust)
- improve compressed mode (slow reads) - improve compressed mode (slow reads)
+3 -3
View File
@@ -22,7 +22,7 @@ full = [
"interface", "interface",
"server", "server",
"store", "store",
"vec", "vecs",
] ]
bundler = ["brk_bundler"] bundler = ["brk_bundler"]
core = ["brk_core"] core = ["brk_core"]
@@ -36,7 +36,7 @@ parser = ["brk_parser"]
interface = ["brk_interface"] interface = ["brk_interface"]
server = ["brk_server"] server = ["brk_server"]
store = ["brk_store"] store = ["brk_store"]
vec = ["brk_vec"] vecs = ["brk_vecs"]
[dependencies] [dependencies]
brk_bundler = { workspace = true, optional = true } brk_bundler = { workspace = true, optional = true }
@@ -52,7 +52,7 @@ brk_parser = { workspace = true, optional = true }
brk_interface = { workspace = true, optional = true } brk_interface = { workspace = true, optional = true }
brk_server = { workspace = true, optional = true } brk_server = { workspace = true, optional = true }
brk_store = { workspace = true, optional = true } brk_store = { workspace = true, optional = true }
brk_vec = { workspace = true, optional = true } brk_vecs = { workspace = true, optional = true }
[package.metadata.docs.rs] [package.metadata.docs.rs]
all-features = true all-features = true
+1 -2
View File
@@ -1,2 +1 @@
cargo build --profile profiling sudo cargo flamegraph --profile profiling --root
flamegraph -- ../../target/profiling/brk
+2 -2
View File
@@ -51,6 +51,6 @@ pub use brk_server as server;
#[doc(inline)] #[doc(inline)]
pub use brk_store as store; pub use brk_store as store;
#[cfg(feature = "vec")] #[cfg(feature = "vecs")]
#[doc(inline)] #[doc(inline)]
pub use brk_vec as vec; pub use brk_vecs as vecs;
+2 -2
View File
@@ -17,14 +17,14 @@ brk_indexer = { workspace = true }
brk_logger = { workspace = true } brk_logger = { workspace = true }
brk_parser = { workspace = true } brk_parser = { workspace = true }
brk_server = { workspace = true } brk_server = { workspace = true }
brk_vec = { workspace = true } brk_vecs = { workspace = true }
clap = { workspace = true } clap = { workspace = true }
clap_derive = { workspace = true } clap_derive = { workspace = true }
color-eyre = { workspace = true } color-eyre = { workspace = true }
log = { workspace = true } log = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
toml = "0.9.1" toml = "0.9.2"
[[bin]] [[bin]]
name = "brk" name = "brk"
+2 -27
View File
@@ -7,14 +7,12 @@ use bitcoincore_rpc::{self, Auth, Client};
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path}; use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
use brk_fetcher::Fetcher; use brk_fetcher::Fetcher;
use brk_server::Website; use brk_server::Website;
use brk_vec::{Computation, Format}; use brk_vecs::{Computation, Format};
use clap::Parser; use clap::Parser;
use clap_derive::Parser; use clap_derive::Parser;
use color_eyre::eyre::eyre; use color_eyre::eyre::eyre;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::services::Services;
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
#[command(version, about)] #[command(version, about)]
pub struct Config { pub struct Config {
@@ -33,11 +31,6 @@ pub struct Config {
#[arg(long, value_name = "PATH")] #[arg(long, value_name = "PATH")]
brkdir: Option<String>, brkdir: Option<String>,
/// Activated services, default: all, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
services: Option<Services>,
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
#[serde(default, deserialize_with = "default_on_error")] #[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)] #[arg(short, long)]
@@ -129,10 +122,6 @@ impl Config {
config_saved.brkdir = Some(brkdir); config_saved.brkdir = Some(brkdir);
} }
if let Some(services) = config_args.services.take() {
config_saved.services = Some(services);
}
if let Some(computation) = config_args.computation.take() { if let Some(computation) = config_args.computation.take() {
config_saved.computation = Some(computation); config_saved.computation = Some(computation);
} }
@@ -298,22 +287,8 @@ impl Config {
.map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref())) .map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref()))
} }
pub fn outputsdir(&self) -> PathBuf {
self.brkdir().join("outputs")
}
pub fn harsdir(&self) -> PathBuf { pub fn harsdir(&self) -> PathBuf {
self.outputsdir().join("hars") self.brkdir().join("hars")
}
pub fn process(&self) -> bool {
self.services
.is_none_or(|m| m == Services::All || m == Services::Processor)
}
pub fn serve(&self) -> bool {
self.services
.is_none_or(|m| m == Services::All || m == Services::Server)
} }
fn path_cookiefile(&self) -> PathBuf { fn path_cookiefile(&self) -> PathBuf {
+2 -1
View File
@@ -1,10 +1,11 @@
#![doc = include_str!("../README.md")]
use std::{fs, thread}; use std::{fs, thread};
use brk_core::{dot_brk_log_path, dot_brk_path}; use brk_core::{dot_brk_log_path, dot_brk_path};
mod config; mod config;
mod run; mod run;
mod services;
use run::*; use run::*;
+27 -42
View File
@@ -13,18 +13,16 @@ pub fn run() -> color_eyre::Result<()> {
let config = Config::import()?; let config = Config::import()?;
let rpc = config.rpc()?; let rpc = config.rpc()?;
let exit = Exit::new(); let exit = Exit::new();
let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc);
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
let format = config.format(); let format = config.format();
let mut indexer = Indexer::forced_import(&config.outputsdir())?; let mut indexer = Indexer::forced_import(&config.brkdir())?;
let wait_for_synced_node = || -> color_eyre::Result<()> { let wait_for_synced_node = |rpc_client: &bitcoincore_rpc::Client| -> color_eyre::Result<()> {
let is_synced = || -> color_eyre::Result<bool> { let is_synced = || -> color_eyre::Result<bool> {
let info = rpc.get_blockchain_info()?; let info = rpc_client.get_blockchain_info()?;
Ok(info.headers == info.blocks) Ok(info.headers == info.blocks)
}; };
@@ -39,7 +37,7 @@ pub fn run() -> color_eyre::Result<()> {
}; };
let mut computer = Computer::forced_import( let mut computer = Computer::forced_import(
&config.outputsdir(), &config.brkdir(),
&indexer, &indexer,
config.computation(), config.computation(),
config.fetcher(), config.fetcher(),
@@ -50,54 +48,41 @@ pub fn run() -> color_eyre::Result<()> {
.enable_all() .enable_all()
.build()? .build()?
.block_on(async { .block_on(async {
let server = if config.serve() { let served_indexer = indexer.clone();
let served_indexer = indexer.clone(); let served_computer = computer.clone();
let served_computer = computer.clone();
let server = Server::new(served_indexer, served_computer, config.website())?; let server = Server::new(served_indexer, served_computer, config.website())?;
let watch = config.watch(); let watch = config.watch();
let mcp = config.mcp(); let mcp = config.mcp();
let opt = Some(tokio::spawn(async move {
server.serve(watch, mcp).await.unwrap();
}));
sleep(Duration::from_secs(1)); tokio::spawn(async move {
server.serve(watch, mcp).await.unwrap();
});
opt sleep(Duration::from_secs(1));
} else {
None
};
if config.process() { loop {
loop { wait_for_synced_node(rpc)?;
wait_for_synced_node()?;
let block_count = rpc.get_block_count()?; let block_count = rpc.get_block_count()?;
info!("{} blocks found.", block_count + 1); info!("{} blocks found.", block_count + 1);
let starting_indexes = let starting_indexes =
indexer.index(&parser, rpc, &exit, config.check_collisions())?; indexer.index(&parser, rpc, &exit, config.check_collisions())?;
computer.compute(&mut indexer, starting_indexes, &exit)?; computer.compute(&mut indexer, starting_indexes, &exit)?;
if let Some(delay) = config.delay() { if let Some(delay) = config.delay() {
sleep(Duration::from_secs(delay)) sleep(Duration::from_secs(delay))
} }
info!("Waiting for new blocks..."); info!("Waiting for new blocks...");
while block_count == rpc.get_block_count()? { while block_count == rpc.get_block_count()? {
sleep(Duration::from_secs(1)) sleep(Duration::from_secs(1))
}
} }
} }
if let Some(handle) = server {
handle.await.unwrap();
}
Ok(())
}) })
} }
-23
View File
@@ -1,23 +0,0 @@
use clap_derive::{Parser, ValueEnum};
use serde::{Deserialize, Serialize};
#[derive(
Default,
Debug,
Clone,
Copy,
Parser,
ValueEnum,
Serialize,
Deserialize,
PartialEq,
Eq,
PartialOrd,
Ord,
)]
pub enum Services {
#[default]
All,
Processor,
Server,
}
+1 -5
View File
@@ -17,13 +17,9 @@ brk_fetcher = { workspace = true }
brk_indexer = { workspace = true } brk_indexer = { workspace = true }
brk_logger = { workspace = true } brk_logger = { workspace = true }
brk_parser = { workspace = true } brk_parser = { workspace = true }
brk_store = { workspace = true } brk_vecs = { workspace = true }
brk_vec = { workspace = true }
color-eyre = { workspace = true } color-eyre = { workspace = true }
derive_deref = { workspace = true } derive_deref = { workspace = true }
either = "1.15.0"
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true } log = { workspace = true }
rayon = { workspace = true } rayon = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
+7 -4
View File
@@ -1,19 +1,18 @@
use std::{path::Path, thread}; use std::{path::Path, thread};
use brk_computer::Computer; use brk_computer::Computer;
use brk_core::{default_bitcoin_path, default_brk_path};
use brk_exit::Exit; use brk_exit::Exit;
use brk_fetcher::Fetcher; use brk_fetcher::Fetcher;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_parser::Parser; use brk_parser::Parser;
use brk_vec::{Computation, Format}; use brk_vecs::{Computation, Format};
pub fn main() -> color_eyre::Result<()> { pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?; color_eyre::install()?;
brk_logger::init(Some(Path::new(".log"))); brk_logger::init(Some(Path::new(".log")));
// let bitcoin_dir = default_bitcoin_path(); // let bitcoin_dir = brk_core::default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin"); let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
@@ -26,7 +25,11 @@ pub fn main() -> color_eyre::Result<()> {
thread::Builder::new() thread::Builder::new()
.stack_size(256 * 1024 * 1024) .stack_size(256 * 1024 * 1024)
.spawn(move || -> color_eyre::Result<()> { .spawn(move || -> color_eyre::Result<()> {
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc); let parser = Parser::new(
bitcoin_dir.join("blocks"),
brk_core::default_brk_path(),
rpc,
);
let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs"); let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs");
let outputs_dir = _outputs_dir.as_path(); let outputs_dir = _outputs_dir.as_path();
@@ -1,26 +1,15 @@
use std::path::Path; use std::{path::Path, sync::Arc};
use brk_core::Version; use brk_core::Version;
use brk_exit::Exit; use brk_exit::Exit;
use brk_fetcher::Fetcher; use brk_fetcher::Fetcher;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format}; use brk_vecs::{AnyCollectableVec, Computation, File, Format};
pub mod blocks;
pub mod cointime;
pub mod constants;
pub mod fetched;
pub mod grouped;
pub mod indexes;
pub mod market;
pub mod mining;
pub mod stateful;
pub mod transactions;
pub use indexes::Indexes;
use log::info; use log::info;
use crate::stores::Stores; use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions};
use super::stateful;
const VERSION: Version = Version::ONE; const VERSION: Version = Version::ONE;
@@ -38,16 +27,19 @@ pub struct Vecs {
} }
impl Vecs { impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn import( pub fn import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
indexer: &Indexer, indexer: &Indexer,
fetch: bool, fetch: bool,
computation: Computation, computation: Computation,
format: Format, format: Format,
fetched_file: &Arc<File>,
states_path: &Path,
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let indexes = indexes::Vecs::forced_import( let indexes = indexes::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
indexer, indexer,
computation, computation,
@@ -56,7 +48,8 @@ impl Vecs {
let fetched = fetch.then(|| { let fetched = fetch.then(|| {
fetched::Vecs::forced_import( fetched::Vecs::forced_import(
path, file,
fetched_file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
computation, computation,
format, format,
@@ -67,43 +60,44 @@ impl Vecs {
Ok(Self { Ok(Self {
blocks: blocks::Vecs::forced_import( blocks: blocks::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
computation, computation,
format, format,
&indexes, &indexes,
)?, )?,
mining: mining::Vecs::forced_import( mining: mining::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
computation, computation,
format, format,
&indexes, &indexes,
)?, )?,
constants: constants::Vecs::forced_import( constants: constants::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
computation, computation,
format, format,
&indexes, &indexes,
)?, )?,
market: market::Vecs::forced_import( market: market::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
computation, computation,
format, format,
&indexes, &indexes,
)?, )?,
stateful: stateful::Vecs::forced_import( stateful: stateful::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
computation, computation,
format, format,
&indexes, &indexes,
fetched.as_ref(), fetched.as_ref(),
states_path,
)?, )?,
transactions: transactions::Vecs::forced_import( transactions: transactions::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
indexer, indexer,
&indexes, &indexes,
@@ -112,7 +106,7 @@ impl Vecs {
fetched.as_ref(), fetched.as_ref(),
)?, )?,
cointime: cointime::Vecs::forced_import( cointime: cointime::Vecs::forced_import(
path, file,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
computation, computation,
format, format,
@@ -130,7 +124,6 @@ impl Vecs {
starting_indexes: brk_indexer::Indexes, starting_indexes: brk_indexer::Indexes,
fetcher: Option<&mut Fetcher>, fetcher: Option<&mut Fetcher>,
exit: &Exit, exit: &Exit,
stores: &mut Stores,
) -> color_eyre::Result<()> { ) -> color_eyre::Result<()> {
info!("Computing indexes..."); info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
@@ -188,7 +181,6 @@ impl Vecs {
&self.market, &self.market,
&mut starting_indexes, &mut starting_indexes,
exit, exit,
stores,
)?; )?;
self.cointime.compute( self.cointime.compute(
@@ -1,4 +1,4 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize, CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
@@ -6,9 +6,9 @@ use brk_core::{
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format}; use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, File, Format};
use crate::vecs::grouped::Source; use crate::grouped::Source;
use super::{ use super::{
Indexes, Indexes,
@@ -34,7 +34,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
computation: Computation, computation: Computation,
format: Format, format: Format,
@@ -42,13 +42,13 @@ impl Vecs {
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
Ok(Self { Ok(Self {
height_to_interval: EagerVec::forced_import( height_to_interval: EagerVec::forced_import(
path, file,
"interval", "interval",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import( timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
path, file,
"timestamp", "timestamp",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -58,7 +58,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(), VecBuilderOptions::default().add_first(),
)?, )?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import( indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
path, file,
"block_interval", "block_interval",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -71,7 +71,7 @@ impl Vecs {
.add_average(), .add_average(),
)?, )?,
indexes_to_block_count: ComputedVecsFromHeight::forced_import( indexes_to_block_count: ComputedVecsFromHeight::forced_import(
path, file,
"block_count", "block_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -81,7 +81,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import( indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
path, file,
"block_weight", "block_weight",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -91,7 +91,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import( indexes_to_block_size: ComputedVecsFromHeight::forced_import(
path, file,
"block_size", "block_size",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -101,13 +101,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
height_to_vbytes: EagerVec::forced_import( height_to_vbytes: EagerVec::forced_import(
path, file,
"vbytes", "vbytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
path, file,
"block_vbytes", "block_vbytes",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -117,13 +117,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
difficultyepoch_to_timestamp: EagerVec::forced_import( difficultyepoch_to_timestamp: EagerVec::forced_import(
path, file,
"timestamp", "timestamp",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
halvingepoch_to_timestamp: EagerVec::forced_import( halvingepoch_to_timestamp: EagerVec::forced_import(
path, file,
"timestamp", "timestamp",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -1,20 +1,17 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version}; use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
use crate::vecs::{
fetched,
grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, Source},
stateful, transactions,
};
use super::{ use super::{
Indexes, Indexes, fetched,
grouped::{ComputedVecsFromHeight, VecBuilderOptions}, grouped::{
indexes, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
Source, VecBuilderOptions,
},
indexes, stateful, transactions,
}; };
const VERSION: Version = Version::ZERO; const VERSION: Version = Version::ZERO;
@@ -49,7 +46,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
computation: Computation, computation: Computation,
format: Format, format: Format,
@@ -60,7 +57,7 @@ impl Vecs {
Ok(Self { Ok(Self {
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import( indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
path, file,
"coinblocks_created", "coinblocks_created",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -70,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import( indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
path, file,
"coinblocks_stored", "coinblocks_stored",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -80,7 +77,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_liveliness: ComputedVecsFromHeight::forced_import( indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
path, file,
"liveliness", "liveliness",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -90,7 +87,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import( indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
path, file,
"vaultedness", "vaultedness",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -100,7 +97,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import( indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
path, file,
"activity_to_vaultedness_ratio", "activity_to_vaultedness_ratio",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -110,7 +107,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import( indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
path, file,
"vaulted_supply", "vaulted_supply",
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -121,7 +118,7 @@ impl Vecs {
indexes, indexes,
)?, )?,
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import( indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
path, file,
"active_supply", "active_supply",
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -132,7 +129,7 @@ impl Vecs {
indexes, indexes,
)?, )?,
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import( indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
path, file,
"thermo_cap", "thermo_cap",
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -142,7 +139,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import( indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
path, file,
"investor_cap", "investor_cap",
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -152,7 +149,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import( indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
path, file,
"vaulted_cap", "vaulted_cap",
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -162,7 +159,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_active_cap: ComputedVecsFromHeight::forced_import( indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
path, file,
"active_cap", "active_cap",
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -172,7 +169,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import( indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
path, file,
"vaulted_price", "vaulted_price",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -182,7 +179,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path, file,
"vaulted_price", "vaulted_price",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -192,7 +189,7 @@ impl Vecs {
true, true,
)?, )?,
indexes_to_active_price: ComputedVecsFromHeight::forced_import( indexes_to_active_price: ComputedVecsFromHeight::forced_import(
path, file,
"active_price", "active_price",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -202,7 +199,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path, file,
"active_price", "active_price",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -212,7 +209,7 @@ impl Vecs {
true, true,
)?, )?,
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import( indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
path, file,
"true_market_mean", "true_market_mean",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -222,7 +219,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import( indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path, file,
"true_market_mean", "true_market_mean",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -232,7 +229,7 @@ impl Vecs {
true, true,
)?, )?,
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import( indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
path, file,
"cointime_value_destroyed", "cointime_value_destroyed",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -242,7 +239,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import( indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
path, file,
"cointime_value_created", "cointime_value_created",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -252,7 +249,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import( indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
path, file,
"cointime_value_stored", "cointime_value_stored",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -262,7 +259,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import( indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
path, file,
"cointime_price", "cointime_price",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -272,7 +269,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import( indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
path, file,
"cointime_cap", "cointime_cap",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -282,7 +279,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path, file,
"cointime_price", "cointime_price",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -305,7 +302,7 @@ impl Vecs {
stateful: &stateful::Vecs, stateful: &stateful::Vecs,
exit: &Exit, exit: &Exit,
) -> color_eyre::Result<()> { ) -> color_eyre::Result<()> {
let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply; let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply;
self.indexes_to_coinblocks_created.compute_all( self.indexes_to_coinblocks_created.compute_all(
indexer, indexer,
@@ -323,7 +320,7 @@ impl Vecs {
)?; )?;
let indexes_to_coinblocks_destroyed = let indexes_to_coinblocks_destroyed =
&stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed; &stateful.utxo_cohorts.all.1.indexes_to_coinblocks_destroyed;
self.indexes_to_coinblocks_stored.compute_all( self.indexes_to_coinblocks_stored.compute_all(
indexer, indexer,
@@ -433,7 +430,7 @@ impl Vecs {
if let Some(fetched) = fetched { if let Some(fetched) = fetched {
let realized_cap = stateful let realized_cap = stateful
.utxo_vecs .utxo_cohorts
.all .all
.1 .1
.height_to_realized_cap .height_to_realized_cap
@@ -441,7 +438,7 @@ impl Vecs {
.unwrap(); .unwrap();
let realized_price = stateful let realized_price = stateful
.utxo_vecs .utxo_cohorts
.all .all
.1 .1
.indexes_to_realized_price .indexes_to_realized_price
@@ -1,11 +1,11 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{StoredU8, Version}; use brk_core::{StoredU8, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format}; use brk_vecs::{AnyCollectableVec, AnyVec, Computation, File, Format};
use crate::vecs::grouped::Source; use crate::grouped::Source;
use super::{ use super::{
Indexes, Indexes,
@@ -25,7 +25,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
computation: Computation, computation: Computation,
format: Format, format: Format,
@@ -33,7 +33,7 @@ impl Vecs {
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
Ok(Self { Ok(Self {
constant_0: ComputedVecsFromHeight::forced_import( constant_0: ComputedVecsFromHeight::forced_import(
path, file,
"constant_0", "constant_0",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -43,7 +43,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
constant_1: ComputedVecsFromHeight::forced_import( constant_1: ComputedVecsFromHeight::forced_import(
path, file,
"constant_1", "constant_1",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -53,7 +53,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
constant_50: ComputedVecsFromHeight::forced_import( constant_50: ComputedVecsFromHeight::forced_import(
path, file,
"constant_50", "constant_50",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -63,7 +63,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
constant_100: ComputedVecsFromHeight::forced_import( constant_100: ComputedVecsFromHeight::forced_import(
path, file,
"constant_100", "constant_100",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -1,4 +1,4 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex, Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
@@ -8,11 +8,12 @@ use brk_core::{
use brk_exit::Exit; use brk_exit::Exit;
use brk_fetcher::Fetcher; use brk_fetcher::Fetcher;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, StoredIndex, AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, StoredIndex,
VecIterator,
}; };
use crate::vecs::grouped::Source; use crate::grouped::Source;
use super::{ use super::{
Indexes, Indexes,
@@ -75,103 +76,100 @@ const VERSION_IN_SATS: Version = Version::ZERO;
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
fetched_file: &Arc<File>,
version: Version, version: Version,
computation: Computation, computation: Computation,
format: Format, format: Format,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let mut fetched_path = path.to_owned();
fetched_path.pop();
fetched_path = fetched_path.join("fetched");
Ok(Self { Ok(Self {
dateindex_to_ohlc_in_cents: EagerVec::forced_import( dateindex_to_ohlc_in_cents: EagerVec::forced_import(
&fetched_path, fetched_file,
"ohlc_in_cents", "ohlc_in_cents",
version + Version::ZERO, version + Version::ZERO,
format, format,
)?, )?,
dateindex_to_ohlc: EagerVec::forced_import( dateindex_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_ohlc_in_sats: EagerVec::forced_import( dateindex_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
dateindex_to_close_in_cents: EagerVec::forced_import( dateindex_to_close_in_cents: EagerVec::forced_import(
path, file,
"close_in_cents", "close_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_high_in_cents: EagerVec::forced_import( dateindex_to_high_in_cents: EagerVec::forced_import(
path, file,
"high_in_cents", "high_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_low_in_cents: EagerVec::forced_import( dateindex_to_low_in_cents: EagerVec::forced_import(
path, file,
"low_in_cents", "low_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_open_in_cents: EagerVec::forced_import( dateindex_to_open_in_cents: EagerVec::forced_import(
path, file,
"open_in_cents", "open_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_ohlc_in_cents: EagerVec::forced_import( height_to_ohlc_in_cents: EagerVec::forced_import(
&fetched_path, fetched_file,
"ohlc_in_cents", "ohlc_in_cents",
version + Version::ZERO, version + Version::ZERO,
format, format,
)?, )?,
height_to_ohlc: EagerVec::forced_import( height_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_ohlc_in_sats: EagerVec::forced_import( height_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
height_to_close_in_cents: EagerVec::forced_import( height_to_close_in_cents: EagerVec::forced_import(
path, file,
"close_in_cents", "close_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_high_in_cents: EagerVec::forced_import( height_to_high_in_cents: EagerVec::forced_import(
path, file,
"high_in_cents", "high_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_low_in_cents: EagerVec::forced_import( height_to_low_in_cents: EagerVec::forced_import(
path, file,
"low_in_cents", "low_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_open_in_cents: EagerVec::forced_import( height_to_open_in_cents: EagerVec::forced_import(
path, file,
"open_in_cents", "open_in_cents",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import( timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
path, file,
"open", "open",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -181,7 +179,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(), VecBuilderOptions::default().add_first(),
)?, )?,
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import( timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
path, file,
"high", "high",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -191,7 +189,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(), VecBuilderOptions::default().add_max(),
)?, )?,
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import( timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
path, file,
"low", "low",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -201,7 +199,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(), VecBuilderOptions::default().add_min(),
)?, )?,
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import( timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
path, file,
"close", "close",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -211,7 +209,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import( timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
path, file,
"open_in_sats", "open_in_sats",
Source::Compute, Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -221,7 +219,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(), VecBuilderOptions::default().add_first(),
)?, )?,
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import( timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
path, file,
"high_in_sats", "high_in_sats",
Source::Compute, Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -231,7 +229,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(), VecBuilderOptions::default().add_max(),
)?, )?,
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import( timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
path, file,
"low_in_sats", "low_in_sats",
Source::Compute, Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -241,7 +239,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(), VecBuilderOptions::default().add_min(),
)?, )?,
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import( timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
path, file,
"close_in_sats", "close_in_sats",
Source::Compute, Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -251,143 +249,143 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
path, file,
"open", "open",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
VecBuilderOptions::default().add_first(), VecBuilderOptions::default().add_first(),
)?, )?,
chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
path, file,
"high", "high",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
VecBuilderOptions::default().add_max(), VecBuilderOptions::default().add_max(),
)?, )?,
chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
path, file,
"low", "low",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
VecBuilderOptions::default().add_min(), VecBuilderOptions::default().add_min(),
)?, )?,
chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
path, file,
"close", "close",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
path, file,
"open_in_sats", "open_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
VecBuilderOptions::default().add_first(), VecBuilderOptions::default().add_first(),
)?, )?,
chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
path, file,
"high_in_sats", "high_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
VecBuilderOptions::default().add_max(), VecBuilderOptions::default().add_max(),
)?, )?,
chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
path, file,
"low_in_sats", "low_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
VecBuilderOptions::default().add_min(), VecBuilderOptions::default().add_min(),
)?, )?,
chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import( chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
path, file,
"close_in_sats", "close_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
weekindex_to_ohlc: EagerVec::forced_import( weekindex_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
weekindex_to_ohlc_in_sats: EagerVec::forced_import( weekindex_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
difficultyepoch_to_ohlc: EagerVec::forced_import( difficultyepoch_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import( difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
monthindex_to_ohlc: EagerVec::forced_import( monthindex_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
monthindex_to_ohlc_in_sats: EagerVec::forced_import( monthindex_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
quarterindex_to_ohlc: EagerVec::forced_import( quarterindex_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
quarterindex_to_ohlc_in_sats: EagerVec::forced_import( quarterindex_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
semesterindex_to_ohlc: EagerVec::forced_import( semesterindex_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
semesterindex_to_ohlc_in_sats: EagerVec::forced_import( semesterindex_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
yearindex_to_ohlc: EagerVec::forced_import( yearindex_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
yearindex_to_ohlc_in_sats: EagerVec::forced_import( yearindex_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
)?, )?,
// halvingepoch_to_ohlc: StorableVec::forced_import(path, // halvingepoch_to_ohlc: StorableVec::forced_import(file,
// "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?, // "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
decadeindex_to_ohlc: EagerVec::forced_import( decadeindex_to_ohlc: EagerVec::forced_import(
path, file,
"ohlc", "ohlc",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
decadeindex_to_ohlc_in_sats: EagerVec::forced_import( decadeindex_to_ohlc_in_sats: EagerVec::forced_import(
path, file,
"ohlc_in_sats", "ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO, version + VERSION + VERSION_IN_SATS + Version::ZERO,
format, format,
@@ -456,33 +454,34 @@ impl Vecs {
exit, exit,
)?; )?;
let mut prev = None;
self.dateindex_to_ohlc_in_cents.compute_transform( self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex, starting_indexes.dateindex,
&indexes.dateindex_to_date, &indexes.dateindex_to_date,
|(di, d, this)| { |(di, d, this)| {
let get_prev = || { if prev.is_none() {
this.get_or_read(di, &this.mmap().load()) let i = di.unwrap_to_usize();
.unwrap() prev.replace(if i > 0 {
.unwrap() this.into_iter().unwrap_get_inner_(i - 1)
.into_owned() } else {
}; OHLCCents::default()
});
}
let mut ohlc = if di.unwrap_to_usize() + 100 >= this.len() { let ohlc = if di.unwrap_to_usize() + 100 >= this.len()
fetcher.get_date(d).unwrap_or_else(|_| get_prev()) && let Ok(mut ohlc) = fetcher.get_date(d)
} else { {
get_prev() let prev_open = *prev.as_ref().unwrap().close;
};
if let Some(prev) = di.decremented() {
let prev_open = *this
.get_or_read(prev, &this.mmap().load())
.unwrap()
.unwrap()
.close;
*ohlc.open = prev_open; *ohlc.open = prev_open;
*ohlc.high = (*ohlc.high).max(prev_open); *ohlc.high = (*ohlc.high).max(prev_open);
*ohlc.low = (*ohlc.low).min(prev_open); *ohlc.low = (*ohlc.low).min(prev_open);
} ohlc
} else {
prev.clone().unwrap()
};
prev.replace(ohlc.clone());
(di, ohlc) (di, ohlc)
}, },
exit, exit,
@@ -1,13 +1,13 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{FromCoarserIndex, Result, Version}; use brk_core::{FromCoarserIndex, Result, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation, AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, Format, StoredIndex, ComputedVec, ComputedVecFrom2, File, Format, StoredIndex,
}; };
use crate::vecs::grouped::{EagerVecBuilder, VecBuilderOptions}; use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
use super::ComputedType; use super::ComputedType;
@@ -39,7 +39,7 @@ where
{ {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
version: Version, version: Version,
format: Format, format: Format,
@@ -66,7 +66,7 @@ where
Box::new( Box::new(
ComputedVec::forced_import_or_init_from_2( ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
&maybe_suffix("first"), &maybe_suffix("first"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -91,7 +91,7 @@ where
Box::new( Box::new(
ComputedVec::forced_import_or_init_from_2( ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -100,7 +100,7 @@ where
source source
.as_ref() .as_ref()
.unwrap_or_else(|| { .unwrap_or_else(|| {
dbg!(path, name, I::to_string()); dbg!(file, name, I::to_string());
panic!() panic!()
}) })
.clone() .clone()
@@ -124,7 +124,7 @@ where
Box::new( Box::new(
ComputedVec::forced_import_or_init_from_2( ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
&maybe_suffix("min"), &maybe_suffix("min"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -149,7 +149,7 @@ where
Box::new( Box::new(
ComputedVec::forced_import_or_init_from_2( ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
&maybe_suffix("max"), &maybe_suffix("max"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -174,7 +174,7 @@ where
Box::new( Box::new(
ComputedVec::forced_import_or_init_from_2( ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
&maybe_suffix("average"), &maybe_suffix("average"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -206,7 +206,7 @@ where
Box::new( Box::new(
ComputedVec::forced_import_or_init_from_2( ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
&(if !options.last && !options.average && !options.min && !options.max { &(if !options.last && !options.average && !options.min && !options.max {
name.to_string() name.to_string()
} else { } else {
@@ -241,7 +241,7 @@ where
Box::new( Box::new(
ComputedVec::forced_import_or_init_from_2( ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
&suffix("cumulative"), &suffix("cumulative"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -1,8 +1,10 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{CheckedSub, Result, StoredUsize, Version}; use brk_core::{CheckedSub, Result, StoredUsize, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType}; use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, EagerVec, File, Format, StoredIndex, StoredType,
};
use color_eyre::eyre::ContextCompat; use color_eyre::eyre::ContextCompat;
use crate::utils::get_percentile; use crate::utils::get_percentile;
@@ -37,7 +39,7 @@ where
T: ComputedType, T: ComputedType,
{ {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
version: Version, version: Version,
format: Format, format: Format,
@@ -59,7 +61,7 @@ where
first: options.first.then(|| { first: options.first.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("first"), &maybe_suffix("first"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -69,13 +71,13 @@ where
}), }),
last: options.last.then(|| { last: options.last.then(|| {
Box::new( Box::new(
EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(), EagerVec::forced_import(file, name, version + Version::ZERO, format).unwrap(),
) )
}), }),
min: options.min.then(|| { min: options.min.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("min"), &maybe_suffix("min"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -86,7 +88,7 @@ where
max: options.max.then(|| { max: options.max.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("max"), &maybe_suffix("max"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -97,7 +99,7 @@ where
median: options.median.then(|| { median: options.median.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("median"), &maybe_suffix("median"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -108,7 +110,7 @@ where
average: options.average.then(|| { average: options.average.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("average"), &maybe_suffix("average"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -119,7 +121,7 @@ where
sum: options.sum.then(|| { sum: options.sum.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&(if !options.last && !options.average && !options.min && !options.max { &(if !options.last && !options.average && !options.min && !options.max {
name.to_string() name.to_string()
} else { } else {
@@ -134,7 +136,7 @@ where
cumulative: options.cumulative.then(|| { cumulative: options.cumulative.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("cumulative"), &suffix("cumulative"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -145,7 +147,7 @@ where
_90p: options._90p.then(|| { _90p: options._90p.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("90p"), &maybe_suffix("90p"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -156,7 +158,7 @@ where
_75p: options._75p.then(|| { _75p: options._75p.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("75p"), &maybe_suffix("75p"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -167,7 +169,7 @@ where
_25p: options._25p.then(|| { _25p: options._25p.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("25p"), &maybe_suffix("25p"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -178,7 +180,7 @@ where
_10p: options._10p.then(|| { _10p: options._10p.then(|| {
Box::new( Box::new(
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&maybe_suffix("10p"), &maybe_suffix("10p"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -309,7 +311,7 @@ where
.inspect_err(|_| { .inspect_err(|_| {
dbg!( dbg!(
&values, &values,
max.path(), max.name(),
first_indexes.name(), first_indexes.name(),
first_index, first_index,
count_indexes.name(), count_indexes.name(),
@@ -1,4 +1,4 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex, DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex,
@@ -6,11 +6,11 @@ use brk_core::{
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
}; };
use crate::vecs::{Indexes, grouped::ComputedVecBuilder, indexes}; use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions}; use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions};
@@ -37,7 +37,7 @@ where
{ {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
source: Source<DateIndex, T>, source: Source<DateIndex, T>,
version: Version, version: Version,
@@ -47,11 +47,11 @@ where
options: VecBuilderOptions, options: VecBuilderOptions,
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let dateindex = source.is_compute().then(|| { let dateindex = source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap() EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
}); });
let dateindex_extra = EagerVecBuilder::forced_import( let dateindex_extra = EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -64,7 +64,7 @@ where
Ok(Self { Ok(Self {
weekindex: ComputedVecBuilder::forced_import( weekindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -75,7 +75,7 @@ where
options.into(), options.into(),
)?, )?,
monthindex: ComputedVecBuilder::forced_import( monthindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -86,7 +86,7 @@ where
options.into(), options.into(),
)?, )?,
quarterindex: ComputedVecBuilder::forced_import( quarterindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -97,7 +97,7 @@ where
options.into(), options.into(),
)?, )?,
semesterindex: ComputedVecBuilder::forced_import( semesterindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -108,7 +108,7 @@ where
options.into(), options.into(),
)?, )?,
yearindex: ComputedVecBuilder::forced_import( yearindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -119,7 +119,7 @@ where
options.into(), options.into(),
)?, )?,
decadeindex: ComputedVecBuilder::forced_import( decadeindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -1,4 +1,4 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result,
@@ -6,11 +6,11 @@ use brk_core::{
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
}; };
use crate::vecs::{ use crate::{
Indexes, Indexes,
grouped::{ComputedVecBuilder, Source}, grouped::{ComputedVecBuilder, Source},
indexes, indexes,
@@ -45,7 +45,7 @@ where
{ {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
source: Source<Height, T>, source: Source<Height, T>,
version: Version, version: Version,
@@ -55,11 +55,11 @@ where
options: VecBuilderOptions, options: VecBuilderOptions,
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let height = source.is_compute().then(|| { let height = source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap() EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
}); });
let height_extra = EagerVecBuilder::forced_import( let height_extra = EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -67,7 +67,7 @@ where
)?; )?;
let dateindex = EagerVecBuilder::forced_import( let dateindex = EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -78,7 +78,7 @@ where
Ok(Self { Ok(Self {
weekindex: ComputedVecBuilder::forced_import( weekindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -89,7 +89,7 @@ where
options.into(), options.into(),
)?, )?,
monthindex: ComputedVecBuilder::forced_import( monthindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -100,7 +100,7 @@ where
options.into(), options.into(),
)?, )?,
quarterindex: ComputedVecBuilder::forced_import( quarterindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -111,7 +111,7 @@ where
options.into(), options.into(),
)?, )?,
semesterindex: ComputedVecBuilder::forced_import( semesterindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -122,7 +122,7 @@ where
options.into(), options.into(),
)?, )?,
yearindex: ComputedVecBuilder::forced_import( yearindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -133,7 +133,7 @@ where
options.into(), options.into(),
)?, )?,
decadeindex: ComputedVecBuilder::forced_import( decadeindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -143,12 +143,12 @@ where
indexes.decadeindex_to_decadeindex.boxed_clone(), indexes.decadeindex_to_decadeindex.boxed_clone(),
options.into(), options.into(),
)?, )?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
height, height,
height_extra, height_extra,
dateindex, dateindex,
difficultyepoch: EagerVecBuilder::forced_import( difficultyepoch: EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -1,11 +1,11 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{DifficultyEpoch, Height, Result, Version}; use brk_core::{DifficultyEpoch, Height, Result, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, EagerVec, Format}; use brk_vecs::{AnyCollectableVec, EagerVec, File, Format};
use crate::vecs::{Indexes, indexes}; use crate::{Indexes, indexes};
use super::{ComputedType, EagerVecBuilder, VecBuilderOptions}; use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};
@@ -28,17 +28,17 @@ where
f64: From<T>, f64: From<T>,
{ {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
version: Version, version: Version,
format: Format, format: Format,
options: VecBuilderOptions, options: VecBuilderOptions,
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let height = let height =
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?; EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?;
let height_extra = EagerVecBuilder::forced_import( let height_extra = EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -51,13 +51,13 @@ where
height, height,
height_extra, height_extra,
difficultyepoch: EagerVecBuilder::forced_import( difficultyepoch: EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
options, options,
)?, )?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
}) })
} }
@@ -1,4 +1,4 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex, Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
@@ -6,12 +6,12 @@ use brk_core::{
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec, AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec,
Format, StoredIndex, VecIterator, File, Format, StoredIndex, VecIterator,
}; };
use crate::vecs::{ use crate::{
Indexes, fetched, Indexes, fetched,
grouped::{ComputedVecBuilder, Source}, grouped::{ComputedVecBuilder, Source},
indexes, indexes,
@@ -46,7 +46,7 @@ where
{ {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
source: Source<TxIndex, T>, source: Source<TxIndex, T>,
version: Version, version: Version,
@@ -57,13 +57,13 @@ where
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let txindex = source.is_compute().then(|| { let txindex = source.is_compute().then(|| {
Box::new( Box::new(
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format) EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
.unwrap(), .unwrap(),
) )
}); });
let height = EagerVecBuilder::forced_import( let height = EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -73,7 +73,7 @@ where
let options = options.remove_percentiles(); let options = options.remove_percentiles();
let dateindex = EagerVecBuilder::forced_import( let dateindex = EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -82,7 +82,7 @@ where
Ok(Self { Ok(Self {
weekindex: ComputedVecBuilder::forced_import( weekindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -93,7 +93,7 @@ where
options.into(), options.into(),
)?, )?,
monthindex: ComputedVecBuilder::forced_import( monthindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -104,7 +104,7 @@ where
options.into(), options.into(),
)?, )?,
quarterindex: ComputedVecBuilder::forced_import( quarterindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -115,7 +115,7 @@ where
options.into(), options.into(),
)?, )?,
semesterindex: ComputedVecBuilder::forced_import( semesterindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -126,7 +126,7 @@ where
options.into(), options.into(),
)?, )?,
yearindex: ComputedVecBuilder::forced_import( yearindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -137,7 +137,7 @@ where
options.into(), options.into(),
)?, )?,
decadeindex: ComputedVecBuilder::forced_import( decadeindex: ComputedVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -152,13 +152,13 @@ where
height, height,
dateindex, dateindex,
difficultyepoch: EagerVecBuilder::forced_import( difficultyepoch: EagerVecBuilder::forced_import(
path, file,
name, name,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
options, options,
)?, )?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
}) })
} }
@@ -1,17 +1,14 @@
use std::{f32, path::Path}; use std::{f32, sync::Arc};
use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version}; use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format,
StoredIndex, VecIterator, StoredIndex, VecIterator,
}; };
use crate::{ use crate::{Indexes, fetched, grouped::source::Source, indexes, utils::get_percentile};
utils::get_percentile,
vecs::{Indexes, fetched, grouped::source::Source, indexes},
};
use super::{ComputedVecsFromDateIndex, VecBuilderOptions}; use super::{ComputedVecsFromDateIndex, VecBuilderOptions};
@@ -63,7 +60,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedRatioVecsFromDateIndex { impl ComputedRatioVecsFromDateIndex {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
source: Source<DateIndex, Dollars>, source: Source<DateIndex, Dollars>,
version: Version, version: Version,
@@ -77,7 +74,7 @@ impl ComputedRatioVecsFromDateIndex {
Ok(Self { Ok(Self {
price: source.is_compute().then(|| { price: source.is_compute().then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
name, name,
Source::Compute, Source::Compute,
version + VERSION, version + VERSION,
@@ -89,7 +86,7 @@ impl ComputedRatioVecsFromDateIndex {
.unwrap() .unwrap()
}), }),
ratio: ComputedVecsFromDateIndex::forced_import( ratio: ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio"), &format!("{name}_ratio"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -100,7 +97,7 @@ impl ComputedRatioVecsFromDateIndex {
)?, )?,
ratio_sma: extended.then(|| { ratio_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_sma"), &format!("{name}_ratio_sma"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -113,7 +110,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_1w_sma: extended.then(|| { ratio_1w_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_1w_sma"), &format!("{name}_ratio_1w_sma"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -126,7 +123,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_1m_sma: extended.then(|| { ratio_1m_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_1m_sma"), &format!("{name}_ratio_1m_sma"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -139,7 +136,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_1y_sma: extended.then(|| { ratio_1y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_1y_sma"), &format!("{name}_ratio_1y_sma"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -152,7 +149,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_4y_sma: extended.then(|| { ratio_4y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_4y_sma"), &format!("{name}_ratio_4y_sma"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -165,7 +162,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_1y_sma_momentum_oscillator: extended.then(|| { ratio_1y_sma_momentum_oscillator: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_1y_sma_momentum_oscillator"), &format!("{name}_ratio_1y_sma_momentum_oscillator"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -178,7 +175,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_sd: extended.then(|| { ratio_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_sd"), &format!("{name}_ratio_sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -191,7 +188,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_4y_sd: extended.then(|| { ratio_4y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_4y_sd"), &format!("{name}_ratio_4y_sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -204,7 +201,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_1y_sd: extended.then(|| { ratio_1y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_1y_sd"), &format!("{name}_ratio_1y_sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -217,7 +214,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p99_9: extended.then(|| { ratio_p99_9: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p99_9"), &format!("{name}_ratio_p99_9"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -230,7 +227,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p99_5: extended.then(|| { ratio_p99_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p99_5"), &format!("{name}_ratio_p99_5"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -243,7 +240,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p99: extended.then(|| { ratio_p99: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p99"), &format!("{name}_ratio_p99"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -256,7 +253,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p1: extended.then(|| { ratio_p1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p1"), &format!("{name}_ratio_p1"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -269,7 +266,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p0_5: extended.then(|| { ratio_p0_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p0_5"), &format!("{name}_ratio_p0_5"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -282,7 +279,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p0_1: extended.then(|| { ratio_p0_1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p0_1"), &format!("{name}_ratio_p0_1"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -295,7 +292,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p1sd: extended.then(|| { ratio_p1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p1sd"), &format!("{name}_ratio_p1sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -308,7 +305,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p2sd: extended.then(|| { ratio_p2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p2sd"), &format!("{name}_ratio_p2sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -321,7 +318,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p3sd: extended.then(|| { ratio_p3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p3sd"), &format!("{name}_ratio_p3sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -334,7 +331,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_m1sd: extended.then(|| { ratio_m1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_m1sd"), &format!("{name}_ratio_m1sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -347,7 +344,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_m2sd: extended.then(|| { ratio_m2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_m2sd"), &format!("{name}_ratio_m2sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -360,7 +357,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_m3sd: extended.then(|| { ratio_m3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_m3sd"), &format!("{name}_ratio_m3sd"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -373,7 +370,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p99_9_as_price: extended.then(|| { ratio_p99_9_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p99_9_as_price"), &format!("{name}_ratio_p99_9_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -386,7 +383,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p99_5_as_price: extended.then(|| { ratio_p99_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p99_5_as_price"), &format!("{name}_ratio_p99_5_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -399,7 +396,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p99_as_price: extended.then(|| { ratio_p99_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p99_as_price"), &format!("{name}_ratio_p99_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -412,7 +409,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p1_as_price: extended.then(|| { ratio_p1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p1_as_price"), &format!("{name}_ratio_p1_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -425,7 +422,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p0_5_as_price: extended.then(|| { ratio_p0_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p0_5_as_price"), &format!("{name}_ratio_p0_5_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -438,7 +435,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p0_1_as_price: extended.then(|| { ratio_p0_1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p0_1_as_price"), &format!("{name}_ratio_p0_1_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -451,7 +448,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p1sd_as_price: extended.then(|| { ratio_p1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p1sd_as_price"), &format!("{name}_ratio_p1sd_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -464,7 +461,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p2sd_as_price: extended.then(|| { ratio_p2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p2sd_as_price"), &format!("{name}_ratio_p2sd_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -477,7 +474,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_p3sd_as_price: extended.then(|| { ratio_p3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_p3sd_as_price"), &format!("{name}_ratio_p3sd_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -490,7 +487,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_m1sd_as_price: extended.then(|| { ratio_m1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_m1sd_as_price"), &format!("{name}_ratio_m1sd_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -503,7 +500,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_m2sd_as_price: extended.then(|| { ratio_m2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_m2sd_as_price"), &format!("{name}_ratio_m2sd_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -516,7 +513,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_m3sd_as_price: extended.then(|| { ratio_m3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_m3sd_as_price"), &format!("{name}_ratio_m3sd_as_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -529,7 +526,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_zscore: extended.then(|| { ratio_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_zscore"), &format!("{name}_ratio_zscore"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -542,7 +539,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_4y_zscore: extended.then(|| { ratio_4y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_4y_zscore"), &format!("{name}_ratio_4y_zscore"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -555,7 +552,7 @@ impl ComputedRatioVecsFromDateIndex {
}), }),
ratio_1y_zscore: extended.then(|| { ratio_1y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_ratio_1y_zscore"), &format!("{name}_ratio_1y_zscore"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -1072,6 +1069,10 @@ impl ComputedRatioVecsFromDateIndex {
Ok(()) Ok(())
})?; })?;
drop(sma_iter);
drop(_4y_sma_iter);
drop(_1y_sma_iter);
self.mut_ratio_vecs() self.mut_ratio_vecs()
.into_iter() .into_iter()
.try_for_each(|v| v.safe_flush(exit))?; .try_for_each(|v| v.safe_flush(exit))?;
@@ -1,4 +1,4 @@
use brk_vec::BoxedAnyIterableVec; use brk_vecs::BoxedAnyIterableVec;
#[derive(Clone)] #[derive(Clone)]
pub enum Source<I, T> { pub enum Source<I, T> {
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, Div}; use std::ops::{Add, AddAssign, Div};
use brk_vec::StoredType; use brk_vecs::StoredType;
pub trait ComputedType pub trait ComputedType
where where
@@ -1,11 +1,11 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version}; use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes}; use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
use super::{Source, VecBuilderOptions}; use super::{Source, VecBuilderOptions};
@@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromDateIndex { impl ComputedValueVecsFromDateIndex {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
source: Source<DateIndex, Sats>, source: Source<DateIndex, Sats>,
version: Version, version: Version,
@@ -33,7 +33,7 @@ impl ComputedValueVecsFromDateIndex {
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
Ok(Self { Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import( sats: ComputedVecsFromDateIndex::forced_import(
path, file,
name, name,
source, source,
version + VERSION, version + VERSION,
@@ -43,7 +43,7 @@ impl ComputedValueVecsFromDateIndex {
options, options,
)?, )?,
bitcoin: ComputedVecsFromDateIndex::forced_import( bitcoin: ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_in_btc"), &format!("{name}_in_btc"),
Source::Compute, Source::Compute,
version + VERSION, version + VERSION,
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromDateIndex {
)?, )?,
dollars: compute_dollars.then(|| { dollars: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&format!("{name}_in_usd"), &format!("{name}_in_usd"),
Source::Compute, Source::Compute,
version + VERSION, version + VERSION,
@@ -1,11 +1,11 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version}; use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; use crate::{Indexes, fetched, grouped::Source, indexes};
use super::{ComputedVecsFromHeight, VecBuilderOptions}; use super::{ComputedVecsFromHeight, VecBuilderOptions};
@@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromHeight { impl ComputedValueVecsFromHeight {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
source: Source<Height, Sats>, source: Source<Height, Sats>,
version: Version, version: Version,
@@ -33,7 +33,7 @@ impl ComputedValueVecsFromHeight {
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
Ok(Self { Ok(Self {
sats: ComputedVecsFromHeight::forced_import( sats: ComputedVecsFromHeight::forced_import(
path, file,
name, name,
source, source,
version + VERSION, version + VERSION,
@@ -43,7 +43,7 @@ impl ComputedValueVecsFromHeight {
options, options,
)?, )?,
bitcoin: ComputedVecsFromHeight::forced_import( bitcoin: ComputedVecsFromHeight::forced_import(
path, file,
&format!("{name}_in_btc"), &format!("{name}_in_btc"),
Source::Compute, Source::Compute,
version + VERSION, version + VERSION,
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromHeight {
)?, )?,
dollars: compute_dollars.then(|| { dollars: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&format!("{name}_in_usd"), &format!("{name}_in_usd"),
Source::Compute, Source::Compute,
version + VERSION, version + VERSION,
@@ -1,14 +1,14 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3, AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3,
Format, LazyVecFrom1, StoredIndex, StoredVec, File, Format, LazyVecFrom1, StoredIndex, StoredVec,
}; };
use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; use crate::{Indexes, fetched, grouped::Source, indexes};
use super::{ComputedVecsFromTxindex, VecBuilderOptions}; use super::{ComputedVecsFromTxindex, VecBuilderOptions};
@@ -38,7 +38,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromTxindex { impl ComputedValueVecsFromTxindex {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
source: Source<TxIndex, Sats>, source: Source<TxIndex, Sats>,
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromTxindex {
let name_in_usd = format!("{name}_in_usd"); let name_in_usd = format!("{name}_in_usd");
let sats = ComputedVecsFromTxindex::forced_import( let sats = ComputedVecsFromTxindex::forced_import(
path, file,
name, name,
source.clone(), source.clone(),
version + VERSION, version + VERSION,
@@ -79,7 +79,7 @@ impl ComputedValueVecsFromTxindex {
); );
let bitcoin = ComputedVecsFromTxindex::forced_import( let bitcoin = ComputedVecsFromTxindex::forced_import(
path, file,
&name_in_btc, &name_in_btc,
Source::None, Source::None,
version + VERSION, version + VERSION,
@@ -92,7 +92,7 @@ impl ComputedValueVecsFromTxindex {
let dollars_txindex = fetched.map(|fetched| { let dollars_txindex = fetched.map(|fetched| {
ComputedVecFrom3::forced_import_or_init_from_3( ComputedVecFrom3::forced_import_or_init_from_3(
computation, computation,
path, file,
&name_in_usd, &name_in_usd,
version + VERSION, version + VERSION,
format, format,
@@ -127,7 +127,7 @@ impl ComputedValueVecsFromTxindex {
dollars_txindex, dollars_txindex,
dollars: compute_dollars.then(|| { dollars: compute_dollars.then(|| {
ComputedVecsFromTxindex::forced_import( ComputedVecsFromTxindex::forced_import(
path, file,
&name_in_usd, &name_in_usd,
Source::None, Source::None,
version + VERSION, version + VERSION,
@@ -1,11 +1,11 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version}; use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec}; use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, File, Format, StoredVec};
use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; use crate::{Indexes, fetched, grouped::Source, indexes};
#[derive(Clone)] #[derive(Clone)]
pub struct ComputedHeightValueVecs { pub struct ComputedHeightValueVecs {
@@ -18,7 +18,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedHeightValueVecs { impl ComputedHeightValueVecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
name: &str, name: &str,
source: Source<Height, Sats>, source: Source<Height, Sats>,
version: Version, version: Version,
@@ -27,18 +27,18 @@ impl ComputedHeightValueVecs {
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
Ok(Self { Ok(Self {
sats: source.is_compute().then(|| { sats: source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format) EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
.unwrap() .unwrap()
}), }),
bitcoin: EagerVec::forced_import( bitcoin: EagerVec::forced_import(
path, file,
&format!("{name}_in_btc"), &format!("{name}_in_btc"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dollars: compute_dollars.then(|| { dollars: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&format!("{name}_in_usd"), &format!("{name}_in_usd"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -1,4 +1,4 @@
use std::{ops::Deref, path::Path}; use std::{ops::Deref, sync::Arc};
use brk_core::{ use brk_core::{
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height, Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
@@ -10,13 +10,11 @@ use brk_core::{
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1, AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator, ComputedVecFrom2, EagerVec, File, Format, StoredIndex, VecIterator,
}; };
use crate::vecs::indexes;
const VERSION: Version = Version::ZERO; const VERSION: Version = Version::ZERO;
#[derive(Clone)] #[derive(Clone)]
@@ -99,7 +97,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
indexer: &Indexer, indexer: &Indexer,
computation: Computation, computation: Computation,
@@ -107,7 +105,7 @@ impl Vecs {
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1( let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"outputindex", "outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -117,7 +115,7 @@ impl Vecs {
let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1( let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"inputindex", "inputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -127,7 +125,7 @@ impl Vecs {
let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1( let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"txindex", "txindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -137,7 +135,7 @@ impl Vecs {
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2( let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
"input_count", "input_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -160,7 +158,7 @@ impl Vecs {
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2( let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
"output_count", "output_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -183,7 +181,7 @@ impl Vecs {
let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1( let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2pk33addressindex", "p2pk33addressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -192,7 +190,7 @@ impl Vecs {
)?; )?;
let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1( let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2pk65addressindex", "p2pk65addressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -201,7 +199,7 @@ impl Vecs {
)?; )?;
let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1( let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2pkhaddressindex", "p2pkhaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -210,7 +208,7 @@ impl Vecs {
)?; )?;
let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1( let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2shaddressindex", "p2shaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -219,7 +217,7 @@ impl Vecs {
)?; )?;
let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1( let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2traddressindex", "p2traddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -228,7 +226,7 @@ impl Vecs {
)?; )?;
let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1( let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2wpkhaddressindex", "p2wpkhaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -237,7 +235,7 @@ impl Vecs {
)?; )?;
let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1( let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2wshaddressindex", "p2wshaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -246,7 +244,7 @@ impl Vecs {
)?; )?;
let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1( let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2aaddressindex", "p2aaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -255,7 +253,7 @@ impl Vecs {
)?; )?;
let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1( let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"p2msoutputindex", "p2msoutputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -264,7 +262,7 @@ impl Vecs {
)?; )?;
let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1( let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"emptyoutputindex", "emptyoutputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -273,7 +271,7 @@ impl Vecs {
)?; )?;
let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1( let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"unknownoutputindex", "unknownoutputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -282,7 +280,7 @@ impl Vecs {
)?; )?;
let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1( let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"opreturnindex", "opreturnindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -310,259 +308,259 @@ impl Vecs {
unknownoutputindex_to_unknownoutputindex, unknownoutputindex_to_unknownoutputindex,
dateindex_to_date: EagerVec::forced_import( dateindex_to_date: EagerVec::forced_import(
path, file,
"date", "date",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_dateindex: EagerVec::forced_import( dateindex_to_dateindex: EagerVec::forced_import(
path, file,
"dateindex", "dateindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_first_height: EagerVec::forced_import( dateindex_to_first_height: EagerVec::forced_import(
path, file,
"first_height", "first_height",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_monthindex: EagerVec::forced_import( dateindex_to_monthindex: EagerVec::forced_import(
path, file,
"monthindex", "monthindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_weekindex: EagerVec::forced_import( dateindex_to_weekindex: EagerVec::forced_import(
path, file,
"weekindex", "weekindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
decadeindex_to_decadeindex: EagerVec::forced_import( decadeindex_to_decadeindex: EagerVec::forced_import(
path, file,
"decadeindex", "decadeindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
decadeindex_to_first_yearindex: EagerVec::forced_import( decadeindex_to_first_yearindex: EagerVec::forced_import(
path, file,
"first_yearindex", "first_yearindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import( difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
path, file,
"difficultyepoch", "difficultyepoch",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
difficultyepoch_to_first_height: EagerVec::forced_import( difficultyepoch_to_first_height: EagerVec::forced_import(
path, file,
"first_height", "first_height",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
halvingepoch_to_first_height: EagerVec::forced_import( halvingepoch_to_first_height: EagerVec::forced_import(
path, file,
"first_height", "first_height",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
halvingepoch_to_halvingepoch: EagerVec::forced_import( halvingepoch_to_halvingepoch: EagerVec::forced_import(
path, file,
"halvingepoch", "halvingepoch",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_date: EagerVec::forced_import( height_to_date: EagerVec::forced_import(
path, file,
"date", "date",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_difficultyepoch: EagerVec::forced_import( height_to_difficultyepoch: EagerVec::forced_import(
path, file,
"difficultyepoch", "difficultyepoch",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_halvingepoch: EagerVec::forced_import( height_to_halvingepoch: EagerVec::forced_import(
path, file,
"halvingepoch", "halvingepoch",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_height: EagerVec::forced_import( height_to_height: EagerVec::forced_import(
path, file,
"height", "height",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
monthindex_to_first_dateindex: EagerVec::forced_import( monthindex_to_first_dateindex: EagerVec::forced_import(
path, file,
"first_dateindex", "first_dateindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
monthindex_to_monthindex: EagerVec::forced_import( monthindex_to_monthindex: EagerVec::forced_import(
path, file,
"monthindex", "monthindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
monthindex_to_quarterindex: EagerVec::forced_import( monthindex_to_quarterindex: EagerVec::forced_import(
path, file,
"quarterindex", "quarterindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
monthindex_to_semesterindex: EagerVec::forced_import( monthindex_to_semesterindex: EagerVec::forced_import(
path, file,
"semesterindex", "semesterindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
monthindex_to_yearindex: EagerVec::forced_import( monthindex_to_yearindex: EagerVec::forced_import(
path, file,
"yearindex", "yearindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
quarterindex_to_first_monthindex: EagerVec::forced_import( quarterindex_to_first_monthindex: EagerVec::forced_import(
path, file,
"first_monthindex", "first_monthindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
semesterindex_to_first_monthindex: EagerVec::forced_import( semesterindex_to_first_monthindex: EagerVec::forced_import(
path, file,
"first_monthindex", "first_monthindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
weekindex_to_first_dateindex: EagerVec::forced_import( weekindex_to_first_dateindex: EagerVec::forced_import(
path, file,
"first_dateindex", "first_dateindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
yearindex_to_first_monthindex: EagerVec::forced_import( yearindex_to_first_monthindex: EagerVec::forced_import(
path, file,
"first_monthindex", "first_monthindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
quarterindex_to_quarterindex: EagerVec::forced_import( quarterindex_to_quarterindex: EagerVec::forced_import(
path, file,
"quarterindex", "quarterindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
semesterindex_to_semesterindex: EagerVec::forced_import( semesterindex_to_semesterindex: EagerVec::forced_import(
path, file,
"semesterindex", "semesterindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
weekindex_to_weekindex: EagerVec::forced_import( weekindex_to_weekindex: EagerVec::forced_import(
path, file,
"weekindex", "weekindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
yearindex_to_decadeindex: EagerVec::forced_import( yearindex_to_decadeindex: EagerVec::forced_import(
path, file,
"decadeindex", "decadeindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
yearindex_to_yearindex: EagerVec::forced_import( yearindex_to_yearindex: EagerVec::forced_import(
path, file,
"yearindex", "yearindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_date_fixed: EagerVec::forced_import( height_to_date_fixed: EagerVec::forced_import(
path, file,
"date_fixed", "date_fixed",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_dateindex: EagerVec::forced_import( height_to_dateindex: EagerVec::forced_import(
path, file,
"dateindex", "dateindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
txindex_to_height: EagerVec::forced_import( txindex_to_height: EagerVec::forced_import(
path, file,
"height", "height",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_timestamp_fixed: EagerVec::forced_import( height_to_timestamp_fixed: EagerVec::forced_import(
path, file,
"timestamp_fixed", "timestamp_fixed",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_txindex_count: EagerVec::forced_import( height_to_txindex_count: EagerVec::forced_import(
path, file,
"txindex_count", "txindex_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
dateindex_to_height_count: EagerVec::forced_import( dateindex_to_height_count: EagerVec::forced_import(
path, file,
"height_count", "height_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
weekindex_to_dateindex_count: EagerVec::forced_import( weekindex_to_dateindex_count: EagerVec::forced_import(
path, file,
"dateindex_count", "dateindex_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
difficultyepoch_to_height_count: EagerVec::forced_import( difficultyepoch_to_height_count: EagerVec::forced_import(
path, file,
"height_count", "height_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
monthindex_to_dateindex_count: EagerVec::forced_import( monthindex_to_dateindex_count: EagerVec::forced_import(
path, file,
"dateindex_count", "dateindex_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
quarterindex_to_monthindex_count: EagerVec::forced_import( quarterindex_to_monthindex_count: EagerVec::forced_import(
path, file,
"monthindex_count", "monthindex_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
semesterindex_to_monthindex_count: EagerVec::forced_import( semesterindex_to_monthindex_count: EagerVec::forced_import(
path, file,
"monthindex_count", "monthindex_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
yearindex_to_monthindex_count: EagerVec::forced_import( yearindex_to_monthindex_count: EagerVec::forced_import(
path, file,
"monthindex_count", "monthindex_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
decadeindex_to_yearindex_count: EagerVec::forced_import( decadeindex_to_yearindex_count: EagerVec::forced_import(
path, file,
"yearindex_count", "yearindex_count",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
outputindex_to_txindex: EagerVec::forced_import( outputindex_to_txindex: EagerVec::forced_import(
path, file,
"txindex", "txindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -1206,7 +1204,7 @@ pub struct Indexes {
} }
impl Indexes { impl Indexes {
pub fn update_from_height(&mut self, height: Height, indexes: &indexes::Vecs) { pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
self.indexes.height = height; self.indexes.height = height;
self.dateindex = DateIndex::try_from( self.dateindex = DateIndex::try_from(
indexes indexes
+36 -24
View File
@@ -3,29 +3,38 @@
#![doc = include_str!("../examples/main.rs")] #![doc = include_str!("../examples/main.rs")]
#![doc = "```"] #![doc = "```"]
use std::path::Path; use std::{path::Path, sync::Arc};
use brk_core::Version; use brk_core::Version;
use brk_exit::Exit; use brk_exit::Exit;
use brk_fetcher::Fetcher; use brk_fetcher::Fetcher;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{Computation, Format}; use brk_vecs::{Computation, File, Format, PAGE_SIZE};
use log::info; use log::info;
mod all;
mod blocks;
mod cointime;
mod constants;
mod fetched;
mod grouped;
mod indexes;
mod market;
mod mining;
mod stateful;
mod states; mod states;
mod stores; mod transactions;
mod utils; mod utils;
mod vecs;
use indexes::Indexes;
use states::*; use states::*;
use stores::Stores;
use vecs::Vecs;
#[derive(Clone)] #[derive(Clone)]
pub struct Computer { pub struct Computer {
file: Arc<File>,
fetcher: Option<Fetcher>, fetcher: Option<Fetcher>,
pub vecs: Vecs, pub vecs: all::Vecs,
pub stores: Stores,
} }
const VERSION: Version = Version::ONE; const VERSION: Version = Version::ONE;
@@ -39,23 +48,28 @@ impl Computer {
fetcher: Option<Fetcher>, fetcher: Option<Fetcher>,
format: Format, format: Format,
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
let computed_path = outputs_dir.join("computed");
let states_path = computed_path.join("states");
let file = Arc::new(File::open(&computed_path.join("vecs"))?);
file.set_min_len(PAGE_SIZE * 100_000_000)?;
file.set_min_regions(50_000)?;
let file_fetched = Arc::new(File::open(&outputs_dir.join("fetched/vecs"))?);
Ok(Self { Ok(Self {
vecs: Vecs::import( vecs: all::Vecs::import(
// TODO: Give self.path, join inside import &file,
&outputs_dir.join("vecs/computed"),
VERSION + Version::ZERO, VERSION + Version::ZERO,
indexer, indexer,
fetcher.is_some(), fetcher.is_some(),
computation, computation,
format, format,
)?, &file_fetched,
stores: Stores::import( &states_path,
// TODO: Give self.path, join inside import
&outputs_dir.join("stores"),
VERSION + Version::ZERO,
&indexer.stores.keyspace,
)?, )?,
fetcher, fetcher,
file,
}) })
} }
} }
@@ -68,12 +82,10 @@ impl Computer {
exit: &Exit, exit: &Exit,
) -> color_eyre::Result<()> { ) -> color_eyre::Result<()> {
info!("Computing..."); info!("Computing...");
self.vecs.compute( self.vecs
indexer, .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)?;
starting_indexes, self.file.flush()?;
self.fetcher.as_mut(), self.file.punch_holes()?;
exit, Ok(())
&mut self.stores,
)
} }
} }
File diff suppressed because it is too large Load Diff
@@ -1,11 +1,11 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version}; use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
use crate::vecs::grouped::Source; use crate::grouped::Source;
use super::{ use super::{
Indexes, Indexes,
@@ -24,7 +24,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
computation: Computation, computation: Computation,
format: Format, format: Format,
@@ -32,7 +32,7 @@ impl Vecs {
) -> color_eyre::Result<Self> { ) -> color_eyre::Result<Self> {
Ok(Self { Ok(Self {
indexes_to_difficulty: ComputedVecsFromHeight::forced_import( indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
path, file,
"difficulty", "difficulty",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -42,7 +42,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import( indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
path, file,
"difficultyepoch", "difficultyepoch",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -52,7 +52,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import( indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
path, file,
"halvingepoch", "halvingepoch",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -1,23 +1,21 @@
use std::{ops::Deref, path::Path}; use std::{ops::Deref, path::Path, sync::Arc};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version}; use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator, AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, VecIterator,
}; };
use crate::{ use crate::{
states::AddressCohortState, Indexes, fetched,
vecs::{ grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
Indexes, fetched, indexes, market,
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, stateful::{
indexes, market, common,
stateful::{ r#trait::{CohortVecs, DynCohortVecs},
common,
r#trait::{CohortVecs, DynCohortVecs},
},
}, },
states::AddressCohortState,
}; };
const VERSION: Version = Version::ZERO; const VERSION: Version = Version::ZERO;
@@ -37,7 +35,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
cohort_name: Option<&str>, cohort_name: Option<&str>,
computation: Computation, computation: Computation,
format: Format, format: Format,
@@ -59,13 +57,13 @@ impl Vecs {
compute_dollars, compute_dollars,
)?, )?,
height_to_address_count: EagerVec::forced_import( height_to_address_count: EagerVec::forced_import(
path, file,
&suffix("address_count"), &suffix("address_count"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import( indexes_to_address_count: ComputedVecsFromHeight::forced_import(
path, file,
&suffix("address_count"), &suffix("address_count"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -75,7 +73,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(), VecBuilderOptions::default().add_last(),
)?, )?,
inner: common::Vecs::forced_import( inner: common::Vecs::forced_import(
path, file,
cohort_name, cohort_name,
computation, computation,
format, format,
@@ -1,16 +1,16 @@
use std::path::Path; use std::{path::Path, sync::Arc};
use brk_core::{ use brk_core::{
AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, GroupFilter, Height, AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
Result, Version, Dollars, GroupFilter, Height, Result, Version,
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_vec::{Computation, Format}; use brk_indexer::Indexer;
use brk_vecs::{AnyIterableVec, Computation, File, Format};
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::vecs::{ use crate::{
Indexes, fetched, indexes, Indexes, fetched, indexes, market,
stateful::{ stateful::{
address_cohort, address_cohort,
r#trait::{CohortVecs, DynCohortVecs}, r#trait::{CohortVecs, DynCohortVecs},
@@ -24,7 +24,7 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>);
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
_computation: Computation, _computation: Computation,
format: Format, format: Format,
@@ -36,7 +36,7 @@ impl Vecs {
AddressGroups { AddressGroups {
amount_range: ByAmountRange { amount_range: ByAmountRange {
_0sats: address_cohort::Vecs::forced_import( _0sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_with_0sats"), Some("addrs_with_0sats"),
_computation, _computation,
format, format,
@@ -47,7 +47,7 @@ impl Vecs {
true, true,
)?, )?,
_1sat_to_10sats: address_cohort::Vecs::forced_import( _1sat_to_10sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1sat_under_10sats"), Some("addrs_above_1sat_under_10sats"),
_computation, _computation,
format, format,
@@ -58,7 +58,7 @@ impl Vecs {
true, true,
)?, )?,
_10sats_to_100sats: address_cohort::Vecs::forced_import( _10sats_to_100sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10sats_under_100sats"), Some("addrs_above_10sats_under_100sats"),
_computation, _computation,
format, format,
@@ -69,7 +69,7 @@ impl Vecs {
true, true,
)?, )?,
_100sats_to_1k_sats: address_cohort::Vecs::forced_import( _100sats_to_1k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_100sats_under_1k_sats"), Some("addrs_above_100sats_under_1k_sats"),
_computation, _computation,
format, format,
@@ -80,7 +80,7 @@ impl Vecs {
true, true,
)?, )?,
_1k_sats_to_10k_sats: address_cohort::Vecs::forced_import( _1k_sats_to_10k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1k_sats_under_10k_sats"), Some("addrs_above_1k_sats_under_10k_sats"),
_computation, _computation,
format, format,
@@ -91,7 +91,7 @@ impl Vecs {
true, true,
)?, )?,
_10k_sats_to_100k_sats: address_cohort::Vecs::forced_import( _10k_sats_to_100k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10k_sats_under_100k_sats"), Some("addrs_above_10k_sats_under_100k_sats"),
_computation, _computation,
format, format,
@@ -102,7 +102,7 @@ impl Vecs {
true, true,
)?, )?,
_100k_sats_to_1m_sats: address_cohort::Vecs::forced_import( _100k_sats_to_1m_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_100k_sats_under_1m_sats"), Some("addrs_above_100k_sats_under_1m_sats"),
_computation, _computation,
format, format,
@@ -113,7 +113,7 @@ impl Vecs {
true, true,
)?, )?,
_1m_sats_to_10m_sats: address_cohort::Vecs::forced_import( _1m_sats_to_10m_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1m_sats_under_10m_sats"), Some("addrs_above_1m_sats_under_10m_sats"),
_computation, _computation,
format, format,
@@ -124,7 +124,7 @@ impl Vecs {
true, true,
)?, )?,
_10m_sats_to_1btc: address_cohort::Vecs::forced_import( _10m_sats_to_1btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10m_sats_under_1btc"), Some("addrs_above_10m_sats_under_1btc"),
_computation, _computation,
format, format,
@@ -135,7 +135,7 @@ impl Vecs {
true, true,
)?, )?,
_1btc_to_10btc: address_cohort::Vecs::forced_import( _1btc_to_10btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1btc_under_10btc"), Some("addrs_above_1btc_under_10btc"),
_computation, _computation,
format, format,
@@ -146,7 +146,7 @@ impl Vecs {
true, true,
)?, )?,
_10btc_to_100btc: address_cohort::Vecs::forced_import( _10btc_to_100btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10btc_under_100btc"), Some("addrs_above_10btc_under_100btc"),
_computation, _computation,
format, format,
@@ -157,7 +157,7 @@ impl Vecs {
true, true,
)?, )?,
_100btc_to_1k_btc: address_cohort::Vecs::forced_import( _100btc_to_1k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_100btc_under_1k_btc"), Some("addrs_above_100btc_under_1k_btc"),
_computation, _computation,
format, format,
@@ -168,7 +168,7 @@ impl Vecs {
true, true,
)?, )?,
_1k_btc_to_10k_btc: address_cohort::Vecs::forced_import( _1k_btc_to_10k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1k_btc_under_10k_btc"), Some("addrs_above_1k_btc_under_10k_btc"),
_computation, _computation,
format, format,
@@ -179,7 +179,7 @@ impl Vecs {
true, true,
)?, )?,
_10k_btc_to_100k_btc: address_cohort::Vecs::forced_import( _10k_btc_to_100k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10k_btc_under_100k_btc"), Some("addrs_above_10k_btc_under_100k_btc"),
_computation, _computation,
format, format,
@@ -190,7 +190,7 @@ impl Vecs {
true, true,
)?, )?,
_100k_btc_or_more: address_cohort::Vecs::forced_import( _100k_btc_or_more: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_100k_btc"), Some("addrs_above_100k_btc"),
_computation, _computation,
format, format,
@@ -203,7 +203,7 @@ impl Vecs {
}, },
lt_amount: ByLowerThanAmount { lt_amount: ByLowerThanAmount {
_10sats: address_cohort::Vecs::forced_import( _10sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_10sats"), Some("addrs_under_10sats"),
_computation, _computation,
format, format,
@@ -214,7 +214,7 @@ impl Vecs {
true, true,
)?, )?,
_100sats: address_cohort::Vecs::forced_import( _100sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_100sats"), Some("addrs_under_100sats"),
_computation, _computation,
format, format,
@@ -225,7 +225,7 @@ impl Vecs {
true, true,
)?, )?,
_1k_sats: address_cohort::Vecs::forced_import( _1k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_1k_sats"), Some("addrs_under_1k_sats"),
_computation, _computation,
format, format,
@@ -236,7 +236,7 @@ impl Vecs {
true, true,
)?, )?,
_10k_sats: address_cohort::Vecs::forced_import( _10k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_10k_sats"), Some("addrs_under_10k_sats"),
_computation, _computation,
format, format,
@@ -247,7 +247,7 @@ impl Vecs {
true, true,
)?, )?,
_100k_sats: address_cohort::Vecs::forced_import( _100k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_100k_sats"), Some("addrs_under_100k_sats"),
_computation, _computation,
format, format,
@@ -258,7 +258,7 @@ impl Vecs {
true, true,
)?, )?,
_1m_sats: address_cohort::Vecs::forced_import( _1m_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_1m_sats"), Some("addrs_under_1m_sats"),
_computation, _computation,
format, format,
@@ -269,7 +269,7 @@ impl Vecs {
true, true,
)?, )?,
_10m_sats: address_cohort::Vecs::forced_import( _10m_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_10m_sats"), Some("addrs_under_10m_sats"),
_computation, _computation,
format, format,
@@ -280,7 +280,7 @@ impl Vecs {
true, true,
)?, )?,
_1btc: address_cohort::Vecs::forced_import( _1btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_1btc"), Some("addrs_under_1btc"),
_computation, _computation,
format, format,
@@ -291,7 +291,7 @@ impl Vecs {
true, true,
)?, )?,
_10btc: address_cohort::Vecs::forced_import( _10btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_10btc"), Some("addrs_under_10btc"),
_computation, _computation,
format, format,
@@ -302,7 +302,7 @@ impl Vecs {
true, true,
)?, )?,
_100btc: address_cohort::Vecs::forced_import( _100btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_100btc"), Some("addrs_under_100btc"),
_computation, _computation,
format, format,
@@ -313,7 +313,7 @@ impl Vecs {
true, true,
)?, )?,
_1k_btc: address_cohort::Vecs::forced_import( _1k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_1k_btc"), Some("addrs_under_1k_btc"),
_computation, _computation,
format, format,
@@ -324,7 +324,7 @@ impl Vecs {
true, true,
)?, )?,
_10k_btc: address_cohort::Vecs::forced_import( _10k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_10k_btc"), Some("addrs_under_10k_btc"),
_computation, _computation,
format, format,
@@ -335,7 +335,7 @@ impl Vecs {
true, true,
)?, )?,
_100k_btc: address_cohort::Vecs::forced_import( _100k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_under_100k_btc"), Some("addrs_under_100k_btc"),
_computation, _computation,
format, format,
@@ -348,7 +348,7 @@ impl Vecs {
}, },
ge_amount: ByGreatEqualAmount { ge_amount: ByGreatEqualAmount {
_1sat: address_cohort::Vecs::forced_import( _1sat: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1sat"), Some("addrs_above_1sat"),
_computation, _computation,
format, format,
@@ -359,7 +359,7 @@ impl Vecs {
true, true,
)?, )?,
_10sats: address_cohort::Vecs::forced_import( _10sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10sats"), Some("addrs_above_10sats"),
_computation, _computation,
format, format,
@@ -370,7 +370,7 @@ impl Vecs {
true, true,
)?, )?,
_100sats: address_cohort::Vecs::forced_import( _100sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_100sats"), Some("addrs_above_100sats"),
_computation, _computation,
format, format,
@@ -381,7 +381,7 @@ impl Vecs {
true, true,
)?, )?,
_1k_sats: address_cohort::Vecs::forced_import( _1k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1k_sats"), Some("addrs_above_1k_sats"),
_computation, _computation,
format, format,
@@ -392,7 +392,7 @@ impl Vecs {
true, true,
)?, )?,
_10k_sats: address_cohort::Vecs::forced_import( _10k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10k_sats"), Some("addrs_above_10k_sats"),
_computation, _computation,
format, format,
@@ -403,7 +403,7 @@ impl Vecs {
true, true,
)?, )?,
_100k_sats: address_cohort::Vecs::forced_import( _100k_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_100k_sats"), Some("addrs_above_100k_sats"),
_computation, _computation,
format, format,
@@ -414,7 +414,7 @@ impl Vecs {
true, true,
)?, )?,
_1m_sats: address_cohort::Vecs::forced_import( _1m_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1m_sats"), Some("addrs_above_1m_sats"),
_computation, _computation,
format, format,
@@ -425,7 +425,7 @@ impl Vecs {
true, true,
)?, )?,
_10m_sats: address_cohort::Vecs::forced_import( _10m_sats: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10m_sats"), Some("addrs_above_10m_sats"),
_computation, _computation,
format, format,
@@ -436,7 +436,7 @@ impl Vecs {
true, true,
)?, )?,
_1btc: address_cohort::Vecs::forced_import( _1btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1btc"), Some("addrs_above_1btc"),
_computation, _computation,
format, format,
@@ -447,7 +447,7 @@ impl Vecs {
true, true,
)?, )?,
_10btc: address_cohort::Vecs::forced_import( _10btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10btc"), Some("addrs_above_10btc"),
_computation, _computation,
format, format,
@@ -458,7 +458,7 @@ impl Vecs {
true, true,
)?, )?,
_100btc: address_cohort::Vecs::forced_import( _100btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_100btc"), Some("addrs_above_100btc"),
_computation, _computation,
format, format,
@@ -469,7 +469,7 @@ impl Vecs {
true, true,
)?, )?,
_1k_btc: address_cohort::Vecs::forced_import( _1k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_1k_btc"), Some("addrs_above_1k_btc"),
_computation, _computation,
format, format,
@@ -480,7 +480,7 @@ impl Vecs {
true, true,
)?, )?,
_10k_btc: address_cohort::Vecs::forced_import( _10k_btc: address_cohort::Vecs::forced_import(
path, file,
Some("addrs_above_10k_btc"), Some("addrs_above_10k_btc"),
_computation, _computation,
format, format,
@@ -535,16 +535,61 @@ impl Vecs {
}) })
.collect::<Vec<_>>(), .collect::<Vec<_>>(),
] ]
.into_par_iter() .into_iter()
.flatten() .flatten()
.try_for_each(|(vecs, stateful)| { .try_for_each(|(vecs, stateful)| {
vecs.compute_from_stateful(starting_indexes, &stateful, exit) vecs.compute_from_stateful(starting_indexes, &stateful, exit)
}) })
} }
pub fn compute_rest_part1(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.as_mut_vecs().into_iter().try_for_each(|(_, v)| {
v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
})
}
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl AnyIterableVec<DateIndex, Bitcoin>,
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
self.0.as_boxed_mut_vecs().into_iter().try_for_each(|v| {
v.into_iter().try_for_each(|(_, v)| {
v.compute_rest_part2(
indexer,
indexes,
fetched,
starting_indexes,
market,
height_to_supply,
dateindex_to_supply,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
})
})
}
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.as_mut_separate_vecs() self.as_mut_separate_vecs()
.par_iter_mut() .into_iter()
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit)) .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))
} }
} }
@@ -1,8 +1,8 @@
use brk_core::{ByAddressType, Height}; use brk_core::{ByAddressType, Height};
use brk_vec::VecIterator; use brk_vecs::VecIterator;
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount; use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
#[derive(Debug, Default, Deref, DerefMut)] #[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(ByAddressType<usize>); pub struct AddressTypeToAddressCount(ByAddressType<usize>);
@@ -1,9 +1,9 @@
use brk_core::{ByAddressType, Height, Result, StoredUsize}; use brk_core::{ByAddressType, Height, Result, StoredUsize};
use brk_exit::Exit; use brk_exit::Exit;
use brk_vec::EagerVec; use brk_vecs::EagerVec;
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount; use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
#[derive(Debug, Clone, Deref, DerefMut)] #[derive(Debug, Clone, Deref, DerefMut)]
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredUsize>>); pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredUsize>>);
@@ -1,9 +1,9 @@
use brk_core::{ByAddressType, StoredUsize}; use brk_core::{ByAddressType, StoredUsize};
use brk_exit::Exit; use brk_exit::Exit;
use brk_vec::AnyCollectableVec; use brk_vecs::AnyCollectableVec;
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use crate::vecs::{ use crate::{
Indexes, grouped::ComputedVecsFromHeight, indexes, Indexes, grouped::ComputedVecsFromHeight, indexes,
stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount, stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
}; };
@@ -0,0 +1,24 @@
use std::collections::BTreeSet;
use brk_core::TypeIndex;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToTypeIndexSet(ByAddressType<BTreeSet<TypeIndex>>);
impl Default for AddressTypeToTypeIndexSet {
fn default() -> Self {
Self(ByAddressType {
p2pk65: BTreeSet::default(),
p2pk33: BTreeSet::default(),
p2pkh: BTreeSet::default(),
p2sh: BTreeSet::default(),
p2wpkh: BTreeSet::default(),
p2wsh: BTreeSet::default(),
p2tr: BTreeSet::default(),
p2a: BTreeSet::default(),
})
}
}
@@ -29,10 +29,6 @@ impl<T> AddressTypeToTypeIndexTree<T> {
mem::swap(own, other); mem::swap(own, other);
} }
} }
pub fn unwrap(self) -> ByAddressType<BTreeMap<TypeIndex, T>> {
self.0
}
} }
impl<T> Default for AddressTypeToTypeIndexTree<T> { impl<T> Default for AddressTypeToTypeIndexTree<T> {
@@ -0,0 +1,57 @@
use std::mem;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
impl<T> AddressTypeToVec<T> {
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
self
}
pub fn merge_mut(&mut self, mut other: Self) {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
}
fn merge_(own: &mut Vec<T>, other: &mut Vec<T>) {
if own.len() >= other.len() {
own.append(other);
} else {
other.append(own);
mem::swap(own, other);
}
}
}
impl<T> Default for AddressTypeToVec<T> {
fn default() -> Self {
Self(ByAddressType {
p2pk65: vec![],
p2pk33: vec![],
p2pkh: vec![],
p2sh: vec![],
p2wpkh: vec![],
p2wsh: vec![],
p2tr: vec![],
p2a: vec![],
})
}
}
@@ -1,26 +1,23 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
Bitcoin, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredF64, StoredUsize, Version, Bitcoin, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredF64, StoredUsize, Version,
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec, AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec,
Format, VecIterator, File, Format, VecIterator,
}; };
use crate::{ use crate::{
states::CohortState, Indexes, fetched,
vecs::{ grouped::{
Indexes, fetched, ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
grouped::{ ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions,
ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex,
ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
Source, VecBuilderOptions,
},
indexes, market,
}, },
indexes, market,
states::CohortState,
}; };
const VERSION: Version = Version::ZERO; const VERSION: Version = Version::ZERO;
@@ -131,7 +128,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
cohort_name: Option<&str>, cohort_name: Option<&str>,
computation: Computation, computation: Computation,
format: Format, format: Format,
@@ -149,7 +146,7 @@ impl Vecs {
let dateindex_to_supply_in_profit = compute_dollars.then(|| { let dateindex_to_supply_in_profit = compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_profit"), &suffix("supply_in_profit"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -159,7 +156,7 @@ impl Vecs {
let dateindex_to_supply_even = compute_dollars.then(|| { let dateindex_to_supply_even = compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_even"), &suffix("supply_even"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -169,7 +166,7 @@ impl Vecs {
let dateindex_to_supply_in_loss = compute_dollars.then(|| { let dateindex_to_supply_in_loss = compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_loss"), &suffix("supply_in_loss"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -179,7 +176,7 @@ impl Vecs {
let dateindex_to_unrealized_profit = compute_dollars.then(|| { let dateindex_to_unrealized_profit = compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("unrealized_profit"), &suffix("unrealized_profit"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -189,7 +186,7 @@ impl Vecs {
let dateindex_to_unrealized_loss = compute_dollars.then(|| { let dateindex_to_unrealized_loss = compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("unrealized_loss"), &suffix("unrealized_loss"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -200,7 +197,7 @@ impl Vecs {
Ok(Self { Ok(Self {
height_to_supply_in_profit: compute_dollars.then(|| { height_to_supply_in_profit: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_profit"), &suffix("supply_in_profit"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -209,7 +206,7 @@ impl Vecs {
}), }),
indexes_to_supply_in_profit: compute_dollars.then(|| { indexes_to_supply_in_profit: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import( ComputedValueVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_in_profit"), &suffix("supply_in_profit"),
dateindex_to_supply_in_profit.as_ref().map(|v | v.boxed_clone()).into(), dateindex_to_supply_in_profit.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -224,7 +221,7 @@ impl Vecs {
dateindex_to_supply_in_profit, dateindex_to_supply_in_profit,
height_to_supply_even: compute_dollars.then(|| { height_to_supply_even: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_even"), &suffix("supply_even"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -233,7 +230,7 @@ impl Vecs {
}), }),
indexes_to_supply_even: compute_dollars.then(|| { indexes_to_supply_even: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import( ComputedValueVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_even"), &suffix("supply_even"),
dateindex_to_supply_even.as_ref().map(|v | v.boxed_clone()).into(), dateindex_to_supply_even.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -248,7 +245,7 @@ impl Vecs {
dateindex_to_supply_even, dateindex_to_supply_even,
height_to_supply_in_loss: compute_dollars.then(|| { height_to_supply_in_loss: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_loss"), &suffix("supply_in_loss"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -257,7 +254,7 @@ impl Vecs {
}), }),
indexes_to_supply_in_loss: compute_dollars.then(|| { indexes_to_supply_in_loss: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import( ComputedValueVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_in_loss"), &suffix("supply_in_loss"),
dateindex_to_supply_in_loss.as_ref().map(|v | v.boxed_clone()).into(), dateindex_to_supply_in_loss.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -272,7 +269,7 @@ impl Vecs {
dateindex_to_supply_in_loss, dateindex_to_supply_in_loss,
height_to_unrealized_profit: compute_dollars.then(|| { height_to_unrealized_profit: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("unrealized_profit"), &suffix("unrealized_profit"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -281,7 +278,7 @@ impl Vecs {
}), }),
indexes_to_unrealized_profit: compute_dollars.then(|| { indexes_to_unrealized_profit: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("unrealized_profit"), &suffix("unrealized_profit"),
dateindex_to_unrealized_profit.as_ref().map(|v | v.boxed_clone()).into(), dateindex_to_unrealized_profit.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -295,7 +292,7 @@ impl Vecs {
dateindex_to_unrealized_profit, dateindex_to_unrealized_profit,
height_to_unrealized_loss: compute_dollars.then(|| { height_to_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("unrealized_loss"), &suffix("unrealized_loss"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -304,7 +301,7 @@ impl Vecs {
}), }),
height_to_min_price_paid: compute_dollars.then(|| { height_to_min_price_paid: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("min_price_paid"), &suffix("min_price_paid"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -313,7 +310,7 @@ impl Vecs {
}), }),
height_to_max_price_paid: compute_dollars.then(|| { height_to_max_price_paid: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("max_price_paid"), &suffix("max_price_paid"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -322,7 +319,7 @@ impl Vecs {
}), }),
indexes_to_unrealized_loss: compute_dollars.then(|| { indexes_to_unrealized_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("unrealized_loss"), &suffix("unrealized_loss"),
dateindex_to_unrealized_loss.as_ref().map(|v | v.boxed_clone()).into(), dateindex_to_unrealized_loss.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -336,7 +333,7 @@ impl Vecs {
dateindex_to_unrealized_loss, dateindex_to_unrealized_loss,
height_to_realized_cap: compute_dollars.then(|| { height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("realized_cap"), &suffix("realized_cap"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -345,7 +342,7 @@ impl Vecs {
}), }),
indexes_to_realized_cap: compute_dollars.then(|| { indexes_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("realized_cap"), &suffix("realized_cap"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -358,7 +355,7 @@ impl Vecs {
}), }),
indexes_to_min_price_paid: compute_dollars.then(|| { indexes_to_min_price_paid: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("min_price_paid"), &suffix("min_price_paid"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -371,7 +368,7 @@ impl Vecs {
}), }),
indexes_to_max_price_paid: compute_dollars.then(|| { indexes_to_max_price_paid: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("max_price_paid"), &suffix("max_price_paid"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -383,13 +380,13 @@ impl Vecs {
.unwrap() .unwrap()
}), }),
height_to_supply: EagerVec::forced_import( height_to_supply: EagerVec::forced_import(
path, file,
&suffix("supply"), &suffix("supply"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_supply_value: ComputedHeightValueVecs::forced_import( height_to_supply_value: ComputedHeightValueVecs::forced_import(
path, file,
&suffix("supply"), &suffix("supply"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -397,7 +394,7 @@ impl Vecs {
compute_dollars, compute_dollars,
)?, )?,
indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import(
path, file,
&suffix("supply"), &suffix("supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -408,13 +405,13 @@ impl Vecs {
indexes, indexes,
)?, )?,
height_to_utxo_count: EagerVec::forced_import( height_to_utxo_count: EagerVec::forced_import(
path, file,
&suffix("utxo_count"), &suffix("utxo_count"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path, file,
&suffix("utxo_count"), &suffix("utxo_count"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -425,7 +422,7 @@ impl Vecs {
)?, )?,
indexes_to_realized_price: compute_dollars.then(|| { indexes_to_realized_price: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("realized_price"), &suffix("realized_price"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -438,7 +435,7 @@ impl Vecs {
}), }),
indexes_to_realized_price_extra: compute_dollars.then(|| { indexes_to_realized_price_extra: compute_dollars.then(|| {
ComputedRatioVecsFromDateIndex::forced_import( ComputedRatioVecsFromDateIndex::forced_import(
path, file,
&suffix("realized_price"), &suffix("realized_price"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -451,7 +448,7 @@ impl Vecs {
}), }),
height_to_realized_profit: compute_dollars.then(|| { height_to_realized_profit: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("realized_profit"), &suffix("realized_profit"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -460,7 +457,7 @@ impl Vecs {
}), }),
indexes_to_realized_profit: compute_dollars.then(|| { indexes_to_realized_profit: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("realized_profit"), &suffix("realized_profit"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -475,7 +472,7 @@ impl Vecs {
}), }),
height_to_realized_loss: compute_dollars.then(|| { height_to_realized_loss: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("realized_loss"), &suffix("realized_loss"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -484,7 +481,7 @@ impl Vecs {
}), }),
indexes_to_realized_loss: compute_dollars.then(|| { indexes_to_realized_loss: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("realized_loss"), &suffix("realized_loss"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -499,7 +496,7 @@ impl Vecs {
}), }),
indexes_to_negative_realized_loss: compute_dollars.then(|| { indexes_to_negative_realized_loss: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("negative_realized_loss"), &suffix("negative_realized_loss"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -512,7 +509,7 @@ impl Vecs {
}), }),
height_to_value_created: compute_dollars.then(|| { height_to_value_created: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("value_created"), &suffix("value_created"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -521,7 +518,7 @@ impl Vecs {
}), }),
indexes_to_value_created: compute_dollars.then(|| { indexes_to_value_created: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("value_created"), &suffix("value_created"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -534,7 +531,7 @@ impl Vecs {
}), }),
indexes_to_realized_value: compute_dollars.then(|| { indexes_to_realized_value: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("realized_value"), &suffix("realized_value"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -547,7 +544,7 @@ impl Vecs {
}), }),
height_to_adjusted_value_created: compute_dollars.then(|| { height_to_adjusted_value_created: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("adjusted_value_created"), &suffix("adjusted_value_created"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -556,7 +553,7 @@ impl Vecs {
}), }),
indexes_to_adjusted_value_created: compute_dollars.then(|| { indexes_to_adjusted_value_created: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("adjusted_value_created"), &suffix("adjusted_value_created"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -569,7 +566,7 @@ impl Vecs {
}), }),
height_to_value_destroyed: compute_dollars.then(|| { height_to_value_destroyed: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("value_destroyed"), &suffix("value_destroyed"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -578,7 +575,7 @@ impl Vecs {
}), }),
indexes_to_value_destroyed: compute_dollars.then(|| { indexes_to_value_destroyed: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("value_destroyed"), &suffix("value_destroyed"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -591,7 +588,7 @@ impl Vecs {
}), }),
height_to_adjusted_value_destroyed: compute_dollars.then(|| { height_to_adjusted_value_destroyed: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("adjusted_value_destroyed"), &suffix("adjusted_value_destroyed"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -600,7 +597,7 @@ impl Vecs {
}), }),
indexes_to_adjusted_value_destroyed: compute_dollars.then(|| { indexes_to_adjusted_value_destroyed: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("adjusted_value_destroyed"), &suffix("adjusted_value_destroyed"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -613,7 +610,7 @@ impl Vecs {
}), }),
indexes_to_realized_cap_30d_change: compute_dollars.then(|| { indexes_to_realized_cap_30d_change: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("realized_cap_30d_change"), &suffix("realized_cap_30d_change"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -626,7 +623,7 @@ impl Vecs {
}), }),
indexes_to_net_realized_profit_and_loss: compute_dollars.then(|| { indexes_to_net_realized_profit_and_loss: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("net_realized_profit_and_loss"), &suffix("net_realized_profit_and_loss"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -641,7 +638,7 @@ impl Vecs {
}), }),
dateindex_to_sell_side_risk_ratio: compute_dollars.then(|| { dateindex_to_sell_side_risk_ratio: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("sell_side_risk_ratio"), &suffix("sell_side_risk_ratio"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -650,7 +647,7 @@ impl Vecs {
}), }),
dateindex_to_spent_output_profit_ratio: compute_dollars.then(|| { dateindex_to_spent_output_profit_ratio: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("spent_output_profit_ratio"), &suffix("spent_output_profit_ratio"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -659,7 +656,7 @@ impl Vecs {
}), }),
dateindex_to_adjusted_spent_output_profit_ratio: compute_dollars.then(|| { dateindex_to_adjusted_spent_output_profit_ratio: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("adjusted_spent_output_profit_ratio"), &suffix("adjusted_spent_output_profit_ratio"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -667,7 +664,7 @@ impl Vecs {
.unwrap() .unwrap()
}), }),
height_to_halved_supply_value: ComputedHeightValueVecs::forced_import( height_to_halved_supply_value: ComputedHeightValueVecs::forced_import(
path, file,
&suffix("halved_supply"), &suffix("halved_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -675,7 +672,7 @@ impl Vecs {
compute_dollars, compute_dollars,
)?, )?,
indexes_to_halved_supply: ComputedValueVecsFromDateIndex::forced_import( indexes_to_halved_supply: ComputedValueVecsFromDateIndex::forced_import(
path, file,
&suffix("halved_supply"), &suffix("halved_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -687,7 +684,7 @@ impl Vecs {
)?, )?,
height_to_negative_unrealized_loss: compute_dollars.then(|| { height_to_negative_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("negative_unrealized_loss"), &suffix("negative_unrealized_loss"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -696,7 +693,7 @@ impl Vecs {
}), }),
indexes_to_negative_unrealized_loss: compute_dollars.then(|| { indexes_to_negative_unrealized_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("negative_unrealized_loss"), &suffix("negative_unrealized_loss"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -709,7 +706,7 @@ impl Vecs {
}), }),
height_to_net_unrealized_profit_and_loss: compute_dollars.then(|| { height_to_net_unrealized_profit_and_loss: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("net_unrealized_profit_and_loss"), &suffix("net_unrealized_profit_and_loss"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -718,7 +715,7 @@ impl Vecs {
}), }),
indexes_to_net_unrealized_profit_and_loss: compute_dollars.then(|| { indexes_to_net_unrealized_profit_and_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("net_unrealized_profit_and_loss"), &suffix("net_unrealized_profit_and_loss"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -732,7 +729,7 @@ impl Vecs {
height_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then( height_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then(
|| { || {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("net_unrealized_profit_and_loss_relative_to_market_cap"), &suffix("net_unrealized_profit_and_loss_relative_to_market_cap"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -743,7 +740,7 @@ impl Vecs {
indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then( indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then(
|| { || {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("net_unrealized_profit_and_loss_relative_to_market_cap"), &suffix("net_unrealized_profit_and_loss_relative_to_market_cap"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -757,7 +754,7 @@ impl Vecs {
), ),
indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| { indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("realized_profit_relative_to_realized_cap"), &suffix("realized_profit_relative_to_realized_cap"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -770,7 +767,7 @@ impl Vecs {
}), }),
indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| { indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("realized_loss_relative_to_realized_cap"), &suffix("realized_loss_relative_to_realized_cap"),
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -784,7 +781,7 @@ impl Vecs {
indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then( indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then(
|| { || {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("net_realized_profit_and_loss_relative_to_realized_cap"), &suffix("net_realized_profit_and_loss_relative_to_realized_cap"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -798,7 +795,7 @@ impl Vecs {
), ),
height_to_supply_even_value: compute_dollars.then(|| { height_to_supply_even_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import( ComputedHeightValueVecs::forced_import(
path, file,
&suffix("supply_even"), &suffix("supply_even"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -809,7 +806,7 @@ impl Vecs {
}), }),
height_to_supply_in_loss_value: compute_dollars.then(|| { height_to_supply_in_loss_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import( ComputedHeightValueVecs::forced_import(
path, file,
&suffix("supply_in_loss"), &suffix("supply_in_loss"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -820,7 +817,7 @@ impl Vecs {
}), }),
height_to_supply_in_profit_value: compute_dollars.then(|| { height_to_supply_in_profit_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import( ComputedHeightValueVecs::forced_import(
path, file,
&suffix("supply_in_profit"), &suffix("supply_in_profit"),
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -831,7 +828,7 @@ impl Vecs {
}), }),
height_to_supply_even_relative_to_own_supply: compute_dollars.then(|| { height_to_supply_even_relative_to_own_supply: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_even_relative_to_own_supply"), &suffix("supply_even_relative_to_own_supply"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -840,7 +837,7 @@ impl Vecs {
}), }),
height_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| { height_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_loss_relative_to_own_supply"), &suffix("supply_in_loss_relative_to_own_supply"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -849,7 +846,7 @@ impl Vecs {
}), }),
height_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| { height_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_profit_relative_to_own_supply"), &suffix("supply_in_profit_relative_to_own_supply"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -858,7 +855,7 @@ impl Vecs {
}), }),
indexes_to_supply_even_relative_to_own_supply: compute_dollars.then(|| { indexes_to_supply_even_relative_to_own_supply: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_even_relative_to_own_supply"), &suffix("supply_even_relative_to_own_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -871,7 +868,7 @@ impl Vecs {
}), }),
indexes_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| { indexes_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_in_loss_relative_to_own_supply"), &suffix("supply_in_loss_relative_to_own_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -884,7 +881,7 @@ impl Vecs {
}), }),
indexes_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| { indexes_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_in_profit_relative_to_own_supply"), &suffix("supply_in_profit_relative_to_own_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -897,7 +894,7 @@ impl Vecs {
}), }),
indexes_to_supply_relative_to_circulating_supply: compute_relative_to_all.then(|| { indexes_to_supply_relative_to_circulating_supply: compute_relative_to_all.then(|| {
ComputedVecsFromHeight::forced_import( ComputedVecsFromHeight::forced_import(
path, file,
&suffix("supply_relative_to_circulating_supply"), &suffix("supply_relative_to_circulating_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -912,7 +909,7 @@ impl Vecs {
&& compute_dollars) && compute_dollars)
.then(|| { .then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_even_relative_to_circulating_supply"), &suffix("supply_even_relative_to_circulating_supply"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -923,7 +920,7 @@ impl Vecs {
&& compute_dollars) && compute_dollars)
.then(|| { .then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_loss_relative_to_circulating_supply"), &suffix("supply_in_loss_relative_to_circulating_supply"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -934,7 +931,7 @@ impl Vecs {
&& compute_dollars) && compute_dollars)
.then(|| { .then(|| {
EagerVec::forced_import( EagerVec::forced_import(
path, file,
&suffix("supply_in_profit_relative_to_circulating_supply"), &suffix("supply_in_profit_relative_to_circulating_supply"),
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
format, format,
@@ -945,7 +942,7 @@ impl Vecs {
&& compute_dollars) && compute_dollars)
.then(|| { .then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_even_relative_to_circulating_supply"), &suffix("supply_even_relative_to_circulating_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -960,7 +957,7 @@ impl Vecs {
&& compute_dollars) && compute_dollars)
.then(|| { .then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_in_loss_relative_to_circulating_supply"), &suffix("supply_in_loss_relative_to_circulating_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -975,7 +972,7 @@ impl Vecs {
&& compute_dollars) && compute_dollars)
.then(|| { .then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("supply_in_profit_relative_to_circulating_supply"), &suffix("supply_in_profit_relative_to_circulating_supply"),
Source::Compute, Source::Compute,
version + VERSION + Version::ONE, version + VERSION + Version::ONE,
@@ -987,19 +984,19 @@ impl Vecs {
.unwrap() .unwrap()
}), }),
height_to_satblocks_destroyed: EagerVec::forced_import( height_to_satblocks_destroyed: EagerVec::forced_import(
path, file,
&suffix("satblocks_destroyed"), &suffix("satblocks_destroyed"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
height_to_satdays_destroyed: EagerVec::forced_import( height_to_satdays_destroyed: EagerVec::forced_import(
path, file,
&suffix("satdays_destroyed"), &suffix("satdays_destroyed"),
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
)?, )?,
indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import( indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import(
path, file,
&suffix("coinblocks_destroyed"), &suffix("coinblocks_destroyed"),
Source::Compute, Source::Compute,
version + VERSION + Version::TWO, version + VERSION + Version::TWO,
@@ -1009,7 +1006,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import( indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import(
path, file,
&suffix("coindays_destroyed"), &suffix("coindays_destroyed"),
Source::Compute, Source::Compute,
version + VERSION + Version::TWO, version + VERSION + Version::TWO,
@@ -1020,7 +1017,7 @@ impl Vecs {
)?, )?,
indexes_to_net_realized_profit_and_loss_cumulative_30d_change: compute_dollars.then(|| { indexes_to_net_realized_profit_and_loss_cumulative_30d_change: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("net_realized_profit_and_loss_cumulative_30d_change"), &suffix("net_realized_profit_and_loss_cumulative_30d_change"),
Source::Compute, Source::Compute,
version + VERSION + Version::new(3), version + VERSION + Version::new(3),
@@ -1033,7 +1030,7 @@ impl Vecs {
}), }),
indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: compute_dollars.then(|| { indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap"), &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap"),
Source::Compute, Source::Compute,
version + VERSION + Version::new(3), version + VERSION + Version::new(3),
@@ -1046,7 +1043,7 @@ impl Vecs {
}), }),
indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: compute_dollars.then(|| { indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import( ComputedVecsFromDateIndex::forced_import(
path, file,
&suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap"), &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap"),
Source::Compute, Source::Compute,
version + VERSION + Version::new(3), version + VERSION + Version::new(3),
@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
use brk_core::Height; use brk_core::Height;
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::AddressTypeToVec; use crate::stateful::AddressTypeToVec;
#[derive(Debug, Default, Deref, DerefMut)] #[derive(Debug, Default, Deref, DerefMut)]
pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>); pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);
@@ -1,6 +1,6 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use brk_vec::{IndexedVec, StoredIndex, StoredType}; use brk_vecs::{StampedVec, StoredIndex, StoredType};
#[derive(Debug)] #[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>); pub struct RangeMap<I, T>(BTreeMap<I, T>);
@@ -20,12 +20,12 @@ where
} }
} }
impl<I, T> From<&IndexedVec<I, T>> for RangeMap<T, I> impl<I, T> From<&StampedVec<I, T>> for RangeMap<T, I>
where where
I: StoredIndex, I: StoredIndex,
T: StoredIndex + StoredType, T: StoredIndex + StoredType,
{ {
fn from(vec: &IndexedVec<I, T>) -> Self { fn from(vec: &StampedVec<I, T>) -> Self {
Self( Self(
vec.into_iter() vec.into_iter()
.map(|(i, v)| (v.into_owned(), i)) .map(|(i, v)| (v.into_owned(), i))
@@ -1,9 +1,9 @@
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec}; use brk_vecs::{AnyCollectableVec, AnyIterableVec};
use crate::vecs::{Indexes, fetched, indexes, market}; use crate::{Indexes, fetched, indexes, market};
pub trait DynCohortVecs: Send + Sync { pub trait DynCohortVecs: Send + Sync {
fn starting_height(&self) -> Height; fn starting_height(&self) -> Height;
@@ -1,18 +1,15 @@
use std::{ops::Deref, path::Path}; use std::{ops::Deref, path::Path, sync::Arc};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format}; use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, File, Format};
use crate::{ use crate::{
UTXOCohortState, Indexes, UTXOCohortState, fetched, indexes, market,
vecs::{ stateful::{
Indexes, fetched, indexes, market, common,
stateful::{ r#trait::{CohortVecs, DynCohortVecs},
common,
r#trait::{CohortVecs, DynCohortVecs},
},
}, },
}; };
@@ -28,7 +25,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
cohort_name: Option<&str>, cohort_name: Option<&str>,
computation: Computation, computation: Computation,
format: Format, format: Format,
@@ -51,7 +48,7 @@ impl Vecs {
)?, )?,
inner: common::Vecs::forced_import( inner: common::Vecs::forced_import(
path, file,
cohort_name, cohort_name,
computation, computation,
format, format,
@@ -0,0 +1,54 @@
use brk_core::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
#[derive(Debug)]
pub enum WithAddressDataSource<T> {
New(T),
FromLoadedAddressDataVec((LoadedAddressIndex, T)),
FromEmptyAddressDataVec((EmptyAddressIndex, T)),
}
impl<T> WithAddressDataSource<T> {
pub fn is_new(&self) -> bool {
matches!(self, Self::New(_))
}
pub fn is_from_emptyaddressdata(&self) -> bool {
matches!(self, Self::FromEmptyAddressDataVec(_))
}
pub fn deref_mut(&mut self) -> &mut T {
match self {
Self::New(v) => v,
Self::FromLoadedAddressDataVec((_, v)) => v,
Self::FromEmptyAddressDataVec((_, v)) => v,
}
}
}
impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
Self::FromLoadedAddressDataVec((i, v.into()))
}
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
Self::FromEmptyAddressDataVec((i, v.into()))
}
}
}
}
impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
Self::FromLoadedAddressDataVec((i, v.into()))
}
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
Self::FromEmptyAddressDataVec((i, v.into()))
}
}
}
}
@@ -1,6 +1,6 @@
use std::path::Path; use std::path::Path;
use brk_core::{AddressData, Dollars, Height, Result, Sats}; use brk_core::{Dollars, Height, LoadedAddressData, Result, Sats};
use crate::SupplyState; use crate::SupplyState;
@@ -35,7 +35,7 @@ impl AddressCohortState {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn send( pub fn send(
&mut self, &mut self,
addressdata: &mut AddressData, addressdata: &mut LoadedAddressData,
value: Sats, value: Sats,
current_price: Option<Dollars>, current_price: Option<Dollars>,
prev_price: Option<Dollars>, prev_price: Option<Dollars>,
@@ -72,7 +72,12 @@ impl AddressCohortState {
Ok(()) Ok(())
} }
pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option<Dollars>) { pub fn receive(
&mut self,
address_data: &mut LoadedAddressData,
value: Sats,
price: Option<Dollars>,
) {
let compute_price = price.is_some(); let compute_price = price.is_some();
let prev_realized_price = compute_price.then(|| address_data.realized_price()); let prev_realized_price = compute_price.then(|| address_data.realized_price());
@@ -96,7 +101,7 @@ impl AddressCohortState {
); );
} }
pub fn add(&mut self, addressdata: &AddressData) { pub fn add(&mut self, addressdata: &LoadedAddressData) {
self.address_count += 1; self.address_count += 1;
self.inner.increment_( self.inner.increment_(
&addressdata.into(), &addressdata.into(),
@@ -105,7 +110,7 @@ impl AddressCohortState {
); );
} }
pub fn subtract(&mut self, addressdata: &AddressData) { pub fn subtract(&mut self, addressdata: &LoadedAddressData) {
self.address_count = self.address_count.checked_sub(1).unwrap(); self.address_count = self.address_count.checked_sub(1).unwrap();
self.inner.decrement_( self.inner.decrement_(
&addressdata.into(), &addressdata.into(),
@@ -223,17 +223,15 @@ impl CohortState {
let update_state = let update_state =
|price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| { |price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
match price.cmp(&current_price) { match price.cmp(&current_price) {
Ordering::Equal => {
state.supply_even += sats;
}
Ordering::Less => { Ordering::Less => {
state.supply_in_profit += sats; state.supply_in_profit += sats;
if price > Dollars::ZERO && current_price > Dollars::ZERO { if price > Dollars::ZERO && current_price > Dollars::ZERO {
let diff = current_price.checked_sub(price).unwrap(); let diff = current_price.checked_sub(price).unwrap();
if diff <= Dollars::ZERO { // Add back once in a while to verify, but generally not needed
dbg!(price, current_price, diff, sats); // if diff <= Dollars::ZERO {
panic!(); // dbg!(price, current_price, diff, sats);
} // panic!();
// }
state.unrealized_profit += diff * sats; state.unrealized_profit += diff * sats;
} }
} }
@@ -241,13 +239,17 @@ impl CohortState {
state.supply_in_loss += sats; state.supply_in_loss += sats;
if price > Dollars::ZERO && current_price > Dollars::ZERO { if price > Dollars::ZERO && current_price > Dollars::ZERO {
let diff = price.checked_sub(current_price).unwrap(); let diff = price.checked_sub(current_price).unwrap();
if diff <= Dollars::ZERO { // Add back once in a while to verify, but generally not needed
dbg!(price, current_price, diff, sats); // if diff <= Dollars::ZERO {
panic!(); // dbg!(price, current_price, diff, sats);
} // panic!();
// }
state.unrealized_loss += diff * sats; state.unrealized_loss += diff * sats;
} }
} }
Ordering::Equal => {
state.supply_even += sats;
}
} }
}; };
@@ -87,12 +87,12 @@ impl PriceToAmount {
self.height = Some(height); self.height = Some(height);
height.write(&self.path_height())?; height.write(&self.path_height())?;
let config = config::standard();
let file = File::create(self.path_state()).inspect_err(|_| { let file = File::create(self.path_state()).inspect_err(|_| {
dbg!(self.path_state()); dbg!(self.path_state());
})?; })?;
let mut writer = BufWriter::new(file); let mut writer = BufWriter::new(file);
encode_into_std_write(&self.state, &mut writer, config)?; encode_into_std_write(&self.state, &mut writer, config::standard())?;
Ok(()) Ok(())
} }
+3 -3
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, SubAssign}; use std::ops::{Add, AddAssign, SubAssign};
use brk_core::{AddressData, CheckedSub, Sats}; use brk_core::{CheckedSub, LoadedAddressData, Sats};
use serde::Serialize; use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -40,8 +40,8 @@ impl SubAssign<&SupplyState> for SupplyState {
} }
} }
impl From<&AddressData> for SupplyState { impl From<&LoadedAddressData> for SupplyState {
fn from(value: &AddressData) -> Self { fn from(value: &LoadedAddressData) -> Self {
Self { Self {
utxos: value.outputs_len as usize, utxos: value.outputs_len as usize,
value: value.amount(), value: value.amount(),
-647
View File
@@ -1,647 +0,0 @@
use std::{path::Path, thread};
use brk_core::{
AddressData, ByAddressType, EmptyAddressData, Height, OutputType, P2AAddressIndex,
P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex,
P2WPKHAddressIndex, P2WSHAddressIndex, Result, TypeIndex, Version,
};
use brk_store::{AnyStore, Store};
use fjall::{PersistMode, TransactionalKeyspace};
use log::info;
use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Stores {
keyspace: TransactionalKeyspace,
pub p2aaddressindex_to_addressdata: Store<P2AAddressIndex, AddressData>,
pub p2aaddressindex_to_emptyaddressdata: Store<P2AAddressIndex, EmptyAddressData>,
pub p2pk33addressindex_to_addressdata: Store<P2PK33AddressIndex, AddressData>,
pub p2pk33addressindex_to_emptyaddressdata: Store<P2PK33AddressIndex, EmptyAddressData>,
pub p2pk65addressindex_to_addressdata: Store<P2PK65AddressIndex, AddressData>,
pub p2pk65addressindex_to_emptyaddressdata: Store<P2PK65AddressIndex, EmptyAddressData>,
pub p2pkhaddressindex_to_addressdata: Store<P2PKHAddressIndex, AddressData>,
pub p2pkhaddressindex_to_emptyaddressdata: Store<P2PKHAddressIndex, EmptyAddressData>,
pub p2shaddressindex_to_addressdata: Store<P2SHAddressIndex, AddressData>,
pub p2shaddressindex_to_emptyaddressdata: Store<P2SHAddressIndex, EmptyAddressData>,
pub p2traddressindex_to_addressdata: Store<P2TRAddressIndex, AddressData>,
pub p2traddressindex_to_emptyaddressdata: Store<P2TRAddressIndex, EmptyAddressData>,
pub p2wpkhaddressindex_to_addressdata: Store<P2WPKHAddressIndex, AddressData>,
pub p2wpkhaddressindex_to_emptyaddressdata: Store<P2WPKHAddressIndex, EmptyAddressData>,
pub p2wshaddressindex_to_addressdata: Store<P2WSHAddressIndex, AddressData>,
pub p2wshaddressindex_to_emptyaddressdata: Store<P2WSHAddressIndex, EmptyAddressData>,
}
impl Stores {
pub fn import(
path: &Path,
version: Version,
keyspace: &TransactionalKeyspace,
) -> color_eyre::Result<Self> {
let (
(p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata),
(p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata),
(p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata),
(p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata),
(p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata),
(p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata),
(p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata),
(p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata),
) = thread::scope(|scope| {
let p2a = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2aaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2aaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pk33 = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pk33addressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk33addressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pk65 = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pk65addressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk65addressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pkh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2sh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2shaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2shaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2tr = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2traddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2traddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2wpkh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2wsh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2wshaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wshaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
(
p2a.join().unwrap(),
p2pk33.join().unwrap(),
p2pk65.join().unwrap(),
p2pkh.join().unwrap(),
p2sh.join().unwrap(),
p2tr.join().unwrap(),
p2wpkh.join().unwrap(),
p2wsh.join().unwrap(),
)
});
Ok(Self {
keyspace: keyspace.clone(),
p2aaddressindex_to_addressdata,
p2aaddressindex_to_emptyaddressdata,
p2pk33addressindex_to_addressdata,
p2pk33addressindex_to_emptyaddressdata,
p2pk65addressindex_to_addressdata,
p2pk65addressindex_to_emptyaddressdata,
p2pkhaddressindex_to_addressdata,
p2pkhaddressindex_to_emptyaddressdata,
p2shaddressindex_to_addressdata,
p2shaddressindex_to_emptyaddressdata,
p2traddressindex_to_addressdata,
p2traddressindex_to_emptyaddressdata,
p2wpkhaddressindex_to_addressdata,
p2wpkhaddressindex_to_emptyaddressdata,
p2wshaddressindex_to_addressdata,
p2wshaddressindex_to_emptyaddressdata,
})
}
pub fn starting_height(&self) -> Height {
self.as_slice()
.into_iter()
.map(|store| store.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
pub fn reset(&mut self) -> Result<()> {
info!("Resetting stores...");
info!("> If it gets stuck here, stop the program and start it again");
self.as_mut_slice()
.into_iter()
.try_for_each(|store| store.reset())?;
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
}
pub fn get_addressdata(
&self,
address_type: OutputType,
type_index: TypeIndex,
) -> Result<Option<AddressData>> {
Ok(match address_type {
OutputType::P2A => self
.p2aaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK33 => self
.p2pk33addressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK65 => self
.p2pk65addressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PKH => self
.p2pkhaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2SH => self
.p2shaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2TR => self
.p2traddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WPKH => self
.p2wpkhaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WSH => self
.p2wshaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
_ => unreachable!(),
})
}
pub fn get_emptyaddressdata(
&self,
address_type: OutputType,
type_index: TypeIndex,
) -> Result<Option<EmptyAddressData>> {
Ok(match address_type {
OutputType::P2A => self
.p2aaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK33 => self
.p2pk33addressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK65 => self
.p2pk65addressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PKH => self
.p2pkhaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2SH => self
.p2shaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2TR => self
.p2traddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WPKH => self
.p2wpkhaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WSH => self
.p2wshaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
_ => unreachable!(),
})
}
pub fn commit(
&mut self,
height: Height,
addresstype_to_typeindex_to_addressdata: AddressTypeToTypeIndexTree<
WithAddressDataSource<AddressData>,
>,
addresstype_to_typeindex_to_emptyaddressdata: AddressTypeToTypeIndexTree<
WithAddressDataSource<EmptyAddressData>,
>,
) -> Result<()> {
let ByAddressType {
p2pk65,
p2pk33,
p2pkh,
p2sh,
p2wpkh,
p2wsh,
p2tr,
p2a,
} = addresstype_to_typeindex_to_addressdata.unwrap();
let ByAddressType {
p2pk65: empty_p2pk65,
p2pk33: empty_p2pk33,
p2pkh: empty_p2pkh,
p2sh: empty_p2sh,
p2wpkh: empty_p2wpkh,
p2wsh: empty_p2wsh,
p2tr: empty_p2tr,
p2a: empty_p2a,
} = addresstype_to_typeindex_to_emptyaddressdata.unwrap();
thread::scope(|s| {
s.spawn(|| {
self.p2aaddressindex_to_addressdata.commit_(
height,
empty_p2a
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2a.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2pk33addressindex_to_addressdata.commit_(
height,
empty_p2pk33
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2pk33.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2pk65addressindex_to_addressdata.commit_(
height,
empty_p2pk65
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2pk65.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2pkhaddressindex_to_addressdata.commit_(
height,
empty_p2pkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2pkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2shaddressindex_to_addressdata.commit_(
height,
empty_p2sh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2sh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2traddressindex_to_addressdata.commit_(
height,
empty_p2tr
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2tr.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2wpkhaddressindex_to_addressdata.commit_(
height,
empty_p2wpkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2wpkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2wshaddressindex_to_addressdata.commit_(
height,
empty_p2wsh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2wsh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
});
thread::scope(|scope| {
scope.spawn(|| {
self.p2aaddressindex_to_emptyaddressdata.commit_(
height,
p2a.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2a.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2pk33addressindex_to_emptyaddressdata.commit_(
height,
p2pk33
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2pk33.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2pk65addressindex_to_emptyaddressdata.commit_(
height,
p2pk65
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2pk65.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2pkhaddressindex_to_emptyaddressdata.commit_(
height,
p2pkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2pkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2shaddressindex_to_emptyaddressdata.commit_(
height,
p2sh.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2sh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2traddressindex_to_emptyaddressdata.commit_(
height,
p2tr.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2tr.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2wpkhaddressindex_to_emptyaddressdata.commit_(
height,
p2wpkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2wpkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2wshaddressindex_to_emptyaddressdata.commit_(
height,
p2wsh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2wsh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
});
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
}
pub fn rotate_memtables(&self) {
self.as_slice()
.into_iter()
.for_each(|store| store.rotate_memtable());
}
pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] {
[
&self.p2aaddressindex_to_addressdata,
&self.p2aaddressindex_to_emptyaddressdata,
&self.p2pk33addressindex_to_addressdata,
&self.p2pk33addressindex_to_emptyaddressdata,
&self.p2pk65addressindex_to_addressdata,
&self.p2pk65addressindex_to_emptyaddressdata,
&self.p2pkhaddressindex_to_addressdata,
&self.p2pkhaddressindex_to_emptyaddressdata,
&self.p2shaddressindex_to_addressdata,
&self.p2shaddressindex_to_emptyaddressdata,
&self.p2traddressindex_to_addressdata,
&self.p2traddressindex_to_emptyaddressdata,
&self.p2wpkhaddressindex_to_addressdata,
&self.p2wpkhaddressindex_to_emptyaddressdata,
&self.p2wshaddressindex_to_addressdata,
&self.p2wshaddressindex_to_emptyaddressdata,
]
}
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 16] {
[
&mut self.p2aaddressindex_to_addressdata,
&mut self.p2aaddressindex_to_emptyaddressdata,
&mut self.p2pk33addressindex_to_addressdata,
&mut self.p2pk33addressindex_to_emptyaddressdata,
&mut self.p2pk65addressindex_to_addressdata,
&mut self.p2pk65addressindex_to_emptyaddressdata,
&mut self.p2pkhaddressindex_to_addressdata,
&mut self.p2pkhaddressindex_to_emptyaddressdata,
&mut self.p2shaddressindex_to_addressdata,
&mut self.p2shaddressindex_to_emptyaddressdata,
&mut self.p2traddressindex_to_addressdata,
&mut self.p2traddressindex_to_emptyaddressdata,
&mut self.p2wpkhaddressindex_to_addressdata,
&mut self.p2wpkhaddressindex_to_emptyaddressdata,
&mut self.p2wshaddressindex_to_addressdata,
&mut self.p2wshaddressindex_to_emptyaddressdata,
]
}
}
@@ -1,4 +1,4 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32, CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
@@ -6,22 +6,18 @@ use brk_core::{
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{ use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec, AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, File, Format, StoredIndex, VecIterator,
}; };
use crate::vecs::grouped::Source; use crate::grouped::{
ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
use super::{ ComputedVecsFromTxindex, Source, VecBuilderOptions,
Indexes, fetched,
grouped::{
ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
ComputedVecsFromTxindex, VecBuilderOptions,
},
indexes,
}; };
use super::{Indexes, fetched, indexes};
const VERSION: Version = Version::ZERO; const VERSION: Version = Version::ZERO;
#[derive(Clone)] #[derive(Clone)]
@@ -90,7 +86,7 @@ pub struct Vecs {
impl Vecs { impl Vecs {
pub fn forced_import( pub fn forced_import(
path: &Path, file: &Arc<File>,
version: Version, version: Version,
indexer: &Indexer, indexer: &Indexer,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
@@ -102,7 +98,7 @@ impl Vecs {
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2( let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
"value", "value",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -129,7 +125,7 @@ impl Vecs {
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2( let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
"weight", "weight",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -157,7 +153,7 @@ impl Vecs {
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1( let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
computation, computation,
path, file,
"vsize", "vsize",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -174,7 +170,7 @@ impl Vecs {
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2( let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
computation, computation,
path, file,
"is_coinbase", "is_coinbase",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -198,7 +194,7 @@ impl Vecs {
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3( let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
computation, computation,
path, file,
"input_value", "input_value",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -234,7 +230,7 @@ impl Vecs {
// let indexes_to_input_value: ComputedVecsFromTxindex<Sats> = // let indexes_to_input_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import( // ComputedVecsFromTxindex::forced_import(
// path, // file,
// "input_value", // "input_value",
// true, // true,
// version + VERSION + Version::ZERO, // version + VERSION + Version::ZERO,
@@ -248,7 +244,7 @@ impl Vecs {
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3( let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation, computation,
path, file,
"output_value", "output_value",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -284,7 +280,7 @@ impl Vecs {
// let indexes_to_output_value: ComputedVecsFromTxindex<Sats> = // let indexes_to_output_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import( // ComputedVecsFromTxindex::forced_import(
// path, // file,
// "output_value", // "output_value",
// true, // true,
// version + VERSION + Version::ZERO, // version + VERSION + Version::ZERO,
@@ -298,7 +294,7 @@ impl Vecs {
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2( let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
computation, computation,
path, file,
"fee", "fee",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -321,7 +317,7 @@ impl Vecs {
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2( let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
computation, computation,
path, file,
"feerate", "feerate",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
format, format,
@@ -341,7 +337,7 @@ impl Vecs {
Ok(Self { Ok(Self {
indexes_to_tx_count: ComputedVecsFromHeight::forced_import( indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
path, file,
"tx_count", "tx_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -356,7 +352,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import( indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
path, file,
"input_count", "input_count",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -371,7 +367,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import( indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
path, file,
"output_count", "output_count",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -386,7 +382,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
path, file,
"tx_v1", "tx_v1",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -396,7 +392,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import( indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
path, file,
"tx_v2", "tx_v2",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -406,7 +402,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import( indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
path, file,
"tx_v3", "tx_v3",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -416,7 +412,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(), VecBuilderOptions::default().add_sum().add_cumulative(),
)?, )?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import( indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
path, file,
"fee", "fee",
indexes, indexes,
Source::Vec(txindex_to_fee.boxed_clone()), Source::Vec(txindex_to_fee.boxed_clone()),
@@ -432,7 +428,7 @@ impl Vecs {
.add_average(), .add_average(),
)?, )?,
indexes_to_feerate: ComputedVecsFromTxindex::forced_import( indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
path, file,
"feerate", "feerate",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -445,7 +441,7 @@ impl Vecs {
.add_average(), .add_average(),
)?, )?,
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import( indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
path, file,
"tx_vsize", "tx_vsize",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -458,7 +454,7 @@ impl Vecs {
.add_average(), .add_average(),
)?, )?,
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import( indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
path, file,
"tx_weight", "tx_weight",
Source::None, Source::None,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -471,7 +467,7 @@ impl Vecs {
.add_average(), .add_average(),
)?, )?,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
path, file,
"subsidy", "subsidy",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -487,7 +483,7 @@ impl Vecs {
indexes, indexes,
)?, )?,
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import( indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
path, file,
"coinbase", "coinbase",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -503,7 +499,7 @@ impl Vecs {
indexes, indexes,
)?, )?,
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import( indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
path, file,
"unclaimed_rewards", "unclaimed_rewards",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -514,7 +510,7 @@ impl Vecs {
indexes, indexes,
)?, )?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import( indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2a_count", "p2a_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -529,7 +525,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import( indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2ms_count", "p2ms_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -544,7 +540,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import( indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2pk33_count", "p2pk33_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -559,7 +555,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import( indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2pk65_count", "p2pk65_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -574,7 +570,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import( indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2pkh_count", "p2pkh_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -589,7 +585,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import( indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2sh_count", "p2sh_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -604,7 +600,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import( indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2tr_count", "p2tr_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -619,7 +615,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import( indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2wpkh_count", "p2wpkh_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -634,7 +630,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import( indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
path, file,
"p2wsh_count", "p2wsh_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -649,7 +645,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import( indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
path, file,
"opreturn_count", "opreturn_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -664,7 +660,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import( indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
path, file,
"unknownoutput_count", "unknownoutput_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -679,7 +675,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import( indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
path, file,
"emptyoutput_count", "emptyoutput_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -694,7 +690,7 @@ impl Vecs {
.add_cumulative(), .add_cumulative(),
)?, )?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import( indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
path, file,
"exact_utxo_count", "exact_utxo_count",
Source::Compute, Source::Compute,
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
@@ -1,47 +0,0 @@
use std::{collections::BTreeSet, mem};
use brk_core::TypeIndex;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToTypeIndexSet(ByAddressType<BTreeSet<TypeIndex>>);
impl AddressTypeToTypeIndexSet {
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
self
}
fn merge_(own: &mut BTreeSet<TypeIndex>, other: &mut BTreeSet<TypeIndex>) {
if own.len() >= other.len() {
own.append(other);
} else {
other.append(own);
mem::swap(own, other);
}
}
}
impl Default for AddressTypeToTypeIndexSet {
fn default() -> Self {
Self(ByAddressType {
p2pk65: BTreeSet::default(),
p2pk33: BTreeSet::default(),
p2pkh: BTreeSet::default(),
p2sh: BTreeSet::default(),
p2wpkh: BTreeSet::default(),
p2wsh: BTreeSet::default(),
p2tr: BTreeSet::default(),
p2a: BTreeSet::default(),
})
}
}
@@ -1,21 +0,0 @@
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
impl<T> Default for AddressTypeToVec<T> {
fn default() -> Self {
Self(ByAddressType {
p2pk65: vec![],
p2pk33: vec![],
p2pkh: vec![],
p2sh: vec![],
p2wpkh: vec![],
p2wsh: vec![],
p2tr: vec![],
p2a: vec![],
})
}
}
@@ -1,62 +0,0 @@
use brk_core::{AddressData, EmptyAddressData};
#[derive(Debug)]
pub enum WithAddressDataSource<T> {
New(T),
FromAddressDataStore(T),
FromEmptyAddressDataStore(T),
}
impl<T> WithAddressDataSource<T> {
pub fn is_new(&self) -> bool {
matches!(self, Self::New(_))
}
pub fn is_from_addressdata(&self) -> bool {
matches!(self, Self::FromAddressDataStore(_))
}
pub fn is_from_emptyaddressdata(&self) -> bool {
matches!(self, Self::FromEmptyAddressDataStore(_))
}
pub fn deref(&self) -> &T {
match self {
Self::New(v) => v,
Self::FromAddressDataStore(v) => v,
Self::FromEmptyAddressDataStore(v) => v,
}
}
pub fn deref_mut(&mut self) -> &mut T {
match self {
Self::New(v) => v,
Self::FromAddressDataStore(v) => v,
Self::FromEmptyAddressDataStore(v) => v,
}
}
}
impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<AddressData> {
fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
WithAddressDataSource::FromEmptyAddressDataStore(v) => {
Self::FromEmptyAddressDataStore(v.into())
}
}
}
}
impl From<WithAddressDataSource<AddressData>> for WithAddressDataSource<EmptyAddressData> {
fn from(value: WithAddressDataSource<AddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
WithAddressDataSource::FromEmptyAddressDataStore(v) => {
Self::FromEmptyAddressDataStore(v.into())
}
}
}
}
+1 -1
View File
@@ -16,7 +16,7 @@ derive_deref = { workspace = true }
fjall = { workspace = true } fjall = { workspace = true }
jiff = { workspace = true } jiff = { workspace = true }
log = { workspace = true } log = { workspace = true }
rapidhash = "1.4.0" rapidhash = "2.0.2"
rlimit = "0.10.2" rlimit = "0.10.2"
serde = { workspace = true } serde = { workspace = true }
serde_bytes = { workspace = true } serde_bytes = { workspace = true }
+3 -1
View File
@@ -35,7 +35,8 @@ pub enum Error {
WrongAddressType, WrongAddressType,
UnindexableDate, UnindexableDate,
String(&'static str), Str(&'static str),
String(String),
} }
impl From<time::SystemTimeError> for Error { impl From<time::SystemTimeError> for Error {
@@ -134,6 +135,7 @@ impl fmt::Display for Error {
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater" "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
), ),
Error::Str(s) => write!(f, "{s}"),
Error::String(s) => write!(f, "{s}"), Error::String(s) => write!(f, "{s}"),
} }
} }
+8
View File
@@ -8,6 +8,14 @@ pub struct AddressGroups<T> {
} }
impl<T> AddressGroups<T> { impl<T> AddressGroups<T> {
pub fn as_boxed_mut_vecs(&mut self) -> Vec<Box<[&mut T]>> {
vec![
Box::new(self.ge_amount.as_mut_vec()),
Box::new(self.amount_range.as_mut_vec()),
Box::new(self.lt_amount.as_mut_vec()),
]
}
pub fn as_mut_vecs(&mut self) -> Vec<&mut T> { pub fn as_mut_vecs(&mut self) -> Vec<&mut T> {
self.ge_amount self.ge_amount
.as_mut_vec() .as_mut_vec()
@@ -19,6 +19,10 @@ pub struct ByAddressType<T> {
} }
impl<T> ByAddressType<T> { impl<T> ByAddressType<T> {
pub fn get_unwrap(&self, address_type: OutputType) -> &T {
self.get(address_type).unwrap()
}
pub fn get(&self, address_type: OutputType) -> Option<&T> { pub fn get(&self, address_type: OutputType) -> Option<&T> {
match address_type { match address_type {
OutputType::P2PK65 => Some(&self.p2pk65), OutputType::P2PK65 => Some(&self.p2pk65),
@@ -167,3 +171,11 @@ where
self.p2a += rhs.p2a; self.p2a += rhs.p2a;
} }
} }
impl<T> ByAddressType<Option<T>> {
pub fn take(&mut self) {
self.as_mut_vec().into_iter().for_each(|opt| {
opt.take();
});
}
}
@@ -0,0 +1,12 @@
#[derive(Debug, Default)]
pub struct ByAnyAddress<T> {
pub loaded: T,
pub empty: T,
}
impl<T> ByAnyAddress<Option<T>> {
pub fn take(&mut self) {
self.loaded.take();
self.empty.take();
}
}
+12 -15
View File
@@ -15,12 +15,11 @@ pub enum GroupFilter {
impl GroupFilter { impl GroupFilter {
pub fn contains(&self, value: usize) -> bool { pub fn contains(&self, value: usize) -> bool {
match self { match self {
GroupFilter::All => true, GroupFilter::Range(r) => r.contains(&value),
GroupFilter::LowerThan(max) => *max > value, GroupFilter::LowerThan(max) => *max > value,
GroupFilter::GreaterOrEqual(min) => *min <= value, GroupFilter::GreaterOrEqual(min) => *min <= value,
GroupFilter::Range(r) => r.contains(&value), GroupFilter::All => true,
GroupFilter::Epoch(_) => false, GroupFilter::Epoch(_) | GroupFilter::Type(_) => false,
GroupFilter::Type(_) => false,
} }
} }
@@ -28,24 +27,22 @@ impl GroupFilter {
match self { match self {
GroupFilter::All => true, GroupFilter::All => true,
GroupFilter::LowerThan(max) => match other { GroupFilter::LowerThan(max) => match other {
GroupFilter::All => false,
GroupFilter::LowerThan(max2) => max >= max2, GroupFilter::LowerThan(max2) => max >= max2,
GroupFilter::Range(range) => range.end <= *max, GroupFilter::Range(range) => range.end <= *max,
GroupFilter::GreaterOrEqual(_) => false, GroupFilter::All
GroupFilter::Epoch(_) => false, | GroupFilter::GreaterOrEqual(_)
GroupFilter::Type(_) => false, | GroupFilter::Epoch(_)
| GroupFilter::Type(_) => false,
}, },
GroupFilter::GreaterOrEqual(min) => match other { GroupFilter::GreaterOrEqual(min) => match other {
GroupFilter::All => false,
GroupFilter::LowerThan(_) => false,
GroupFilter::Range(range) => range.start >= *min, GroupFilter::Range(range) => range.start >= *min,
GroupFilter::GreaterOrEqual(min2) => min <= min2, GroupFilter::GreaterOrEqual(min2) => min <= min2,
GroupFilter::Epoch(_) => false, GroupFilter::All
GroupFilter::Type(_) => false, | GroupFilter::LowerThan(_)
| GroupFilter::Epoch(_)
| GroupFilter::Type(_) => false,
}, },
GroupFilter::Range(_) => false, GroupFilter::Range(_) | GroupFilter::Epoch(_) | GroupFilter::Type(_) => false,
GroupFilter::Epoch(_) => false,
GroupFilter::Type(_) => false,
} }
} }
} }
+2
View File
@@ -2,6 +2,7 @@ mod address;
mod by_address_type; mod by_address_type;
mod by_age_range; mod by_age_range;
mod by_amount_range; mod by_amount_range;
mod by_any_address;
mod by_epoch; mod by_epoch;
mod by_ge_amount; mod by_ge_amount;
mod by_lt_amount; mod by_lt_amount;
@@ -18,6 +19,7 @@ pub use address::*;
pub use by_address_type::*; pub use by_address_type::*;
pub use by_age_range::*; pub use by_age_range::*;
pub use by_amount_range::*; pub use by_amount_range::*;
pub use by_any_address::*;
pub use by_epoch::*; pub use by_epoch::*;
pub use by_ge_amount::*; pub use by_ge_amount::*;
pub use by_lt_amount::*; pub use by_lt_amount::*;
+15
View File
@@ -18,6 +18,21 @@ pub struct UTXOGroups<T> {
} }
impl<T> UTXOGroups<T> { impl<T> UTXOGroups<T> {
pub fn as_boxed_mut_vecs(&mut self) -> Vec<Box<[&mut T]>> {
vec![
Box::new([&mut self.all]),
Box::new(self.term.as_mut_vec()),
Box::new(self.max_age.as_mut_vec()),
Box::new(self.min_age.as_mut_vec()),
Box::new(self.ge_amount.as_mut_vec()),
Box::new(self.age_range.as_mut_vec()),
Box::new(self.epoch.as_mut_vec()),
Box::new(self.amount_range.as_mut_vec()),
Box::new(self.lt_amount.as_mut_vec()),
Box::new(self._type.as_mut_vec()),
]
}
pub fn as_mut_vecs(&mut self) -> Vec<&mut T> { pub fn as_mut_vecs(&mut self) -> Vec<&mut T> {
[&mut self.all] [&mut self.all]
.into_iter() .into_iter()
@@ -1,5 +1,3 @@
use std::hash::Hasher;
use byteview::ByteView; use byteview::ByteView;
use derive_deref::Deref; use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes}; use zerocopy::{FromBytes, IntoBytes};
@@ -25,9 +23,7 @@ pub struct AddressBytesHash([u8; 8]);
impl From<(&AddressBytes, OutputType)> for AddressBytesHash { impl From<(&AddressBytes, OutputType)> for AddressBytesHash {
fn from((address_bytes, outputtype): (&AddressBytes, OutputType)) -> Self { fn from((address_bytes, outputtype): (&AddressBytes, OutputType)) -> Self {
let mut hasher = rapidhash::RapidHasher::default(); let mut slice = rapidhash::v3::rapidhash_v3(address_bytes.as_slice()).to_le_bytes();
hasher.write(address_bytes.as_slice());
let mut slice = hasher.finish().to_le_bytes();
slice[0] = slice[0].wrapping_add(outputtype as u8); slice[0] = slice[0].wrapping_add(outputtype as u8);
Self(slice) Self(slice)
} }
@@ -0,0 +1,61 @@
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{
TypeIndex,
structs::{EmptyAddressIndex, LoadedAddressIndex},
};
const MIN_EMPTY_INDEX: u32 = u32::MAX - 4_000_000_000;
#[derive(
Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout,
)]
pub struct AnyAddressIndex(TypeIndex);
impl AnyAddressIndex {
pub fn to_enum(&self) -> AnyAddressDataIndexEnum {
AnyAddressDataIndexEnum::from(*self)
}
}
impl From<LoadedAddressIndex> for AnyAddressIndex {
fn from(value: LoadedAddressIndex) -> Self {
if u32::from(value) >= MIN_EMPTY_INDEX {
panic!("")
}
Self(*value)
}
}
impl From<EmptyAddressIndex> for AnyAddressIndex {
fn from(value: EmptyAddressIndex) -> Self {
Self(*value + MIN_EMPTY_INDEX)
}
}
impl Serialize for AnyAddressIndex {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
self.to_enum().serialize(serializer)
}
}
#[derive(Debug, Serialize)]
pub enum AnyAddressDataIndexEnum {
Loaded(LoadedAddressIndex),
Empty(EmptyAddressIndex),
}
impl From<AnyAddressIndex> for AnyAddressDataIndexEnum {
fn from(value: AnyAddressIndex) -> Self {
let uvalue = u32::from(value.0);
if uvalue >= MIN_EMPTY_INDEX {
Self::Empty(EmptyAddressIndex::from(uvalue - MIN_EMPTY_INDEX))
} else {
Self::Loaded(LoadedAddressIndex::from(value.0))
}
}
}
@@ -1,22 +1,23 @@
use byteview::ByteView; use byteview::ByteView;
use serde::Serialize;
use zerocopy::{FromBytes, IntoBytes}; use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{AddressData, Sats}; use crate::{LoadedAddressData, Sats};
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)] #[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
pub struct EmptyAddressData { pub struct EmptyAddressData {
pub transfered: Sats, pub transfered: Sats,
} }
impl From<AddressData> for EmptyAddressData { impl From<LoadedAddressData> for EmptyAddressData {
fn from(value: AddressData) -> Self { fn from(value: LoadedAddressData) -> Self {
Self::from(&value) Self::from(&value)
} }
} }
impl From<&AddressData> for EmptyAddressData { impl From<&LoadedAddressData> for EmptyAddressData {
fn from(value: &AddressData) -> Self { fn from(value: &LoadedAddressData) -> Self {
if value.sent != value.received { if value.sent != value.received {
dbg!(&value); dbg!(&value);
panic!("Trying to convert not empty wallet to empty !"); panic!("Trying to convert not empty wallet to empty !");
@@ -0,0 +1,69 @@
use std::ops::Add;
use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Printable, TypeIndex};
#[derive(
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Deref,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct EmptyAddressIndex(TypeIndex);
impl From<TypeIndex> for EmptyAddressIndex {
fn from(value: TypeIndex) -> Self {
Self(value)
}
}
impl From<usize> for EmptyAddressIndex {
fn from(value: usize) -> Self {
Self(TypeIndex::from(value))
}
}
impl From<u32> for EmptyAddressIndex {
fn from(value: u32) -> Self {
Self(TypeIndex::from(value))
}
}
impl From<EmptyAddressIndex> for usize {
fn from(value: EmptyAddressIndex) -> Self {
usize::from(value.0)
}
}
impl Add<usize> for EmptyAddressIndex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs)
}
}
impl CheckedSub<EmptyAddressIndex> for EmptyAddressIndex {
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self)
}
}
impl Printable for EmptyAddressIndex {
fn to_string() -> &'static str {
"emptyaddressindex"
}
fn to_possible_strings() -> &'static [&'static str] {
&["emptyaddr", "emptyaddressindex"]
}
}
@@ -31,7 +31,7 @@ impl From<TypeIndex> for EmptyOutputIndex {
} }
impl From<EmptyOutputIndex> for usize { impl From<EmptyOutputIndex> for usize {
fn from(value: EmptyOutputIndex) -> Self { fn from(value: EmptyOutputIndex) -> Self {
Self::from(*value) Self::from(value.0)
} }
} }
impl From<usize> for EmptyOutputIndex { impl From<usize> for EmptyOutputIndex {
@@ -42,7 +42,7 @@ impl From<usize> for EmptyOutputIndex {
impl Add<usize> for EmptyOutputIndex { impl Add<usize> for EmptyOutputIndex {
type Output = Self; type Output = Self;
fn add(self, rhs: usize) -> Self::Output { fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs) Self(self.0 + rhs)
} }
} }
impl CheckedSub<EmptyOutputIndex> for EmptyOutputIndex { impl CheckedSub<EmptyOutputIndex> for EmptyOutputIndex {
+1 -1
View File
@@ -35,7 +35,7 @@ impl Height {
pub const ZERO: Self = Self(0); pub const ZERO: Self = Self(0);
pub const MAX: Self = Self(u32::MAX); pub const MAX: Self = Self(u32::MAX);
pub fn new(height: u32) -> Self { pub const fn new(height: u32) -> Self {
Self(height) Self(height)
} }
@@ -1,18 +1,21 @@
use byteview::ByteView; use byteview::ByteView;
use serde::Serialize;
use zerocopy::{FromBytes, IntoBytes}; use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats}; use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats};
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)]
#[repr(C)] #[repr(C)]
pub struct AddressData { pub struct LoadedAddressData {
pub sent: Sats, pub sent: Sats,
pub received: Sats, pub received: Sats,
pub realized_cap: Dollars, pub realized_cap: Dollars,
pub outputs_len: u32, pub outputs_len: u32,
padding: u32,
} }
impl AddressData { impl LoadedAddressData {
pub fn amount(&self) -> Sats { pub fn amount(&self) -> Sats {
(u64::from(self.received) - u64::from(self.sent)).into() (u64::from(self.received) - u64::from(self.sent)).into()
} }
@@ -41,7 +44,7 @@ impl AddressData {
pub fn send(&mut self, amount: Sats, previous_price: Option<Dollars>) -> Result<()> { pub fn send(&mut self, amount: Sats, previous_price: Option<Dollars>) -> Result<()> {
if self.amount() < amount { if self.amount() < amount {
return Err(Error::String("Previous_amount smaller than sent amount")); return Err(Error::Str("Previous_amount smaller than sent amount"));
} }
self.sent += amount; self.sent += amount;
self.outputs_len -= 1; self.outputs_len -= 1;
@@ -55,23 +58,24 @@ impl AddressData {
} }
} }
impl From<EmptyAddressData> for AddressData { impl From<EmptyAddressData> for LoadedAddressData {
fn from(value: EmptyAddressData) -> Self { fn from(value: EmptyAddressData) -> Self {
Self::from(&value) Self::from(&value)
} }
} }
impl From<&EmptyAddressData> for AddressData { impl From<&EmptyAddressData> for LoadedAddressData {
fn from(value: &EmptyAddressData) -> Self { fn from(value: &EmptyAddressData) -> Self {
Self { Self {
sent: value.transfered, sent: value.transfered,
received: value.transfered, received: value.transfered,
realized_cap: Dollars::ZERO, realized_cap: Dollars::ZERO,
outputs_len: 0, outputs_len: 0,
padding: 0,
} }
} }
} }
impl From<ByteView> for AddressData { impl From<ByteView> for LoadedAddressData {
fn from(value: ByteView) -> Self { fn from(value: ByteView) -> Self {
Self { Self {
// MUST be same order as impl From<&AddressData> for ByteView // MUST be same order as impl From<&AddressData> for ByteView
@@ -79,16 +83,17 @@ impl From<ByteView> for AddressData {
received: Sats::read_from_bytes(&value[8..16]).unwrap(), received: Sats::read_from_bytes(&value[8..16]).unwrap(),
realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(), realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(),
outputs_len: u32::read_from_bytes(&value[24..]).unwrap(), outputs_len: u32::read_from_bytes(&value[24..]).unwrap(),
padding: 0,
} }
} }
} }
impl From<AddressData> for ByteView { impl From<LoadedAddressData> for ByteView {
fn from(value: AddressData) -> Self { fn from(value: LoadedAddressData) -> Self {
Self::from(&value) Self::from(&value)
} }
} }
impl From<&AddressData> for ByteView { impl From<&LoadedAddressData> for ByteView {
fn from(value: &AddressData) -> Self { fn from(value: &LoadedAddressData) -> Self {
Self::new( Self::new(
&[ &[
value.sent.as_bytes(), value.sent.as_bytes(),
@@ -0,0 +1,67 @@
use std::ops::Add;
use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Printable, TypeIndex};
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct LoadedAddressIndex(TypeIndex);
impl From<TypeIndex> for LoadedAddressIndex {
fn from(value: TypeIndex) -> Self {
Self(value)
}
}
impl From<usize> for LoadedAddressIndex {
fn from(value: usize) -> Self {
Self(TypeIndex::from(value))
}
}
impl From<LoadedAddressIndex> for usize {
fn from(value: LoadedAddressIndex) -> Self {
usize::from(value.0)
}
}
impl From<LoadedAddressIndex> for u32 {
fn from(value: LoadedAddressIndex) -> Self {
u32::from(value.0)
}
}
impl Add<usize> for LoadedAddressIndex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs)
}
}
impl CheckedSub<LoadedAddressIndex> for LoadedAddressIndex {
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self)
}
}
impl Printable for LoadedAddressIndex {
fn to_string() -> &'static str {
"loadedaddressindex"
}
fn to_possible_strings() -> &'static [&'static str] {
&["loadedaddr", "loadedaddressindex"]
}
}
+8 -2
View File
@@ -1,6 +1,6 @@
mod addressbytes; mod addressbytes;
mod addressbyteshash; mod addressbyteshash;
mod addressdata; mod anyaddressindex;
mod bitcoin; mod bitcoin;
mod blockhash; mod blockhash;
mod blockhashprefix; mod blockhashprefix;
@@ -11,11 +11,14 @@ mod decadeindex;
mod difficultyepoch; mod difficultyepoch;
mod dollars; mod dollars;
mod emptyaddressdata; mod emptyaddressdata;
mod emptyaddressindex;
mod emptyoutputindex; mod emptyoutputindex;
mod feerate; mod feerate;
mod halvingepoch; mod halvingepoch;
mod height; mod height;
mod inputindex; mod inputindex;
mod loadedaddressdata;
mod loadedaddressindex;
mod monthindex; mod monthindex;
mod ohlc; mod ohlc;
mod opreturnindex; mod opreturnindex;
@@ -58,7 +61,7 @@ mod yearindex;
pub use addressbytes::*; pub use addressbytes::*;
pub use addressbyteshash::*; pub use addressbyteshash::*;
pub use addressdata::*; pub use anyaddressindex::*;
pub use bitcoin::*; pub use bitcoin::*;
pub use blockhash::*; pub use blockhash::*;
pub use blockhashprefix::*; pub use blockhashprefix::*;
@@ -69,11 +72,14 @@ pub use decadeindex::*;
pub use difficultyepoch::*; pub use difficultyepoch::*;
pub use dollars::*; pub use dollars::*;
pub use emptyaddressdata::*; pub use emptyaddressdata::*;
pub use emptyaddressindex::*;
pub use emptyoutputindex::*; pub use emptyoutputindex::*;
pub use feerate::*; pub use feerate::*;
pub use halvingepoch::*; pub use halvingepoch::*;
pub use height::*; pub use height::*;
pub use inputindex::*; pub use inputindex::*;
pub use loadedaddressdata::*;
pub use loadedaddressindex::*;
pub use monthindex::*; pub use monthindex::*;
pub use ohlc::*; pub use ohlc::*;
pub use opreturnindex::*; pub use opreturnindex::*;
+6
View File
@@ -73,6 +73,12 @@ impl From<TypeIndex> for usize {
} }
} }
impl Add<u32> for TypeIndex {
type Output = Self;
fn add(self, rhs: u32) -> Self::Output {
Self(self.0 + rhs)
}
}
impl Add<usize> for TypeIndex { impl Add<usize> for TypeIndex {
type Output = Self; type Output = Self;
fn add(self, rhs: usize) -> Self::Output { fn add(self, rhs: usize) -> Self::Output {
+2 -2
View File
@@ -5,7 +5,7 @@ pub fn copy_first_4bytes(slice: &[u8]) -> Result<[u8; 4]> {
let mut buf: [u8; 4] = [0; 4]; let mut buf: [u8; 4] = [0; 4];
let buf_len = buf.len(); let buf_len = buf.len();
if slice.len() < buf_len { if slice.len() < buf_len {
return Err(Error::String("Buffer is too small to convert to 8 bytes")); return Err(Error::Str("Buffer is too small to convert to 8 bytes"));
} }
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r; buf[i] = *r;
@@ -18,7 +18,7 @@ pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8]; let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len(); let buf_len = buf.len();
if slice.len() < buf_len { if slice.len() < buf_len {
return Err(Error::String("Buffer is too small to convert to 8 bytes")); return Err(Error::Str("Buffer is too small to convert to 8 bytes"));
} }
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r; buf[i] = *r;
+1 -1
View File
@@ -7,7 +7,7 @@ pub fn setrlimit() -> io::Result<()> {
rlimit::setrlimit( rlimit::setrlimit(
Resource::NOFILE, Resource::NOFILE,
no_file_limit.0.max(250_000), no_file_limit.0.max(10_000),
no_file_limit.1, no_file_limit.1,
)?; )?;
+1
View File
@@ -11,3 +11,4 @@ repository.workspace = true
brk_logger = { workspace = true } brk_logger = { workspace = true }
ctrlc = { version = "3.4.7", features = ["termination"] } ctrlc = { version = "3.4.7", features = ["termination"] }
log = { workspace = true } log = { workspace = true }
parking_lot = { workspace = true }
+2 -2
View File
@@ -8,7 +8,7 @@ fn main() {
brk_logger::init(Some(Path::new(".log"))); brk_logger::init(Some(Path::new(".log")));
exit.block(); let lock = exit.lock();
let mut i = 0; let mut i = 0;
while i < 21 { while i < 21 {
@@ -17,7 +17,7 @@ fn main() {
i += 1; i += 1;
} }
exit.release(); drop(lock);
let mut j = 0; let mut j = 0;
while j < 10 { while j < 10 {
+11 -46
View File
@@ -3,69 +3,34 @@
#![doc = include_str!("../examples/main.rs")] #![doc = include_str!("../examples/main.rs")]
#![doc = "```"] #![doc = "```"]
use std::{ use std::{process::exit, sync::Arc};
process::exit,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
thread::sleep,
time::Duration,
};
use log::info; use log::info;
use parking_lot::{RwLock, RwLockReadGuard};
#[derive(Default, Clone)] #[derive(Default, Clone)]
pub struct Exit { pub struct Exit(Arc<RwLock<()>>);
blocking: Arc<AtomicBool>,
triggered: Arc<AtomicBool>,
}
impl Exit { impl Exit {
pub fn new() -> Self { pub fn new() -> Self {
let s = Self { let arc = Arc::new(RwLock::new(()));
triggered: Arc::new(AtomicBool::new(false)),
blocking: Arc::new(AtomicBool::new(false)),
};
let triggered = s.triggered.clone(); let copy = arc.clone();
let blocking = s.blocking.clone();
let is_blocking = move || blocking.load(Ordering::SeqCst);
ctrlc::set_handler(move || { ctrlc::set_handler(move || {
info!("Exitting..."); if copy.is_locked() {
triggered.store(true, Ordering::SeqCst);
if is_blocking() {
info!("Waiting to exit safely..."); info!("Waiting to exit safely...");
while is_blocking() {
sleep(Duration::from_millis(50));
}
} }
let _lock = copy.write();
info!("Exiting...");
exit(0); exit(0);
}) })
.expect("Error setting Ctrl-C handler"); .expect("Error setting Ctrl-C handler");
s Self(arc)
} }
pub fn block(&self) { pub fn lock(&self) -> RwLockReadGuard<'_, ()> {
self.blocking.store(true, Ordering::SeqCst); self.0.read()
}
pub fn blocked(&self) -> bool {
self.blocking.load(Ordering::SeqCst)
}
pub fn release(&self) {
self.blocking.store(false, Ordering::SeqCst);
}
pub fn triggered(&self) -> bool {
self.triggered.load(Ordering::SeqCst)
} }
} }
+1 -1
View File
@@ -15,7 +15,7 @@ brk_exit = { workspace = true }
brk_logger = { workspace = true } brk_logger = { workspace = true }
brk_parser = { workspace = true } brk_parser = { workspace = true }
brk_store = { workspace = true } brk_store = { workspace = true }
brk_vec = { workspace = true } brk_vecs = { workspace = true }
color-eyre = { workspace = true } color-eyre = { workspace = true }
fjall = { workspace = true } fjall = { workspace = true }
log = { workspace = true } log = { workspace = true }
+41
View File
@@ -0,0 +1,41 @@
use std::{
fs,
path::Path,
thread::sleep,
time::{Duration, Instant},
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::Parser;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
brk_logger::init(Some(Path::new(".log")));
// let bitcoin_dir = brk_core::default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
// let outputs_dir = brk_core::default_brk_path().join("outputs");
let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);
fs::create_dir_all(outputs_dir)?;
let mut indexer = Indexer::forced_import(outputs_dir)?;
loop {
let i = Instant::now();
indexer.index(&parser, rpc, &exit, true)?;
dbg!(i.elapsed());
sleep(Duration::from_secs(5 * 60));
}
}
-34
View File
@@ -1,34 +0,0 @@
use std::{path::Path, time::Instant};
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::Parser;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let i = Instant::now();
brk_logger::init(Some(Path::new(".log")));
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let outputs = Path::new("../../_outputs");
let mut indexer = Indexer::forced_import(outputs)?;
indexer.index(&parser, rpc, &exit, true)?;
dbg!(i.elapsed());
Ok(())
}
+9 -7
View File
@@ -6,7 +6,7 @@ use brk_core::{
Result, TxIndex, TypeIndex, UnknownOutputIndex, Result, TxIndex, TypeIndex, UnknownOutputIndex,
}; };
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::{AnyIndexedVec, AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType}; use brk_vecs::{AnyIterableVec, AnyStampedVec, AnyVec, StampedVec, StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat; use color_eyre::eyre::ContextCompat;
use crate::{Stores, Vecs}; use crate::{Stores, Vecs};
@@ -89,8 +89,10 @@ impl Indexes {
impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
type Error = color_eyre::Report; type Error = color_eyre::Report;
fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result<Self> { fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result<Self> {
// Height at which we wanna start: min last saved + 1 or 0 // Height at which we want to start: min last saved + 1 or 0
let starting_height = vecs.starting_height().min(stores.starting_height()); let vecs_starting_height = vecs.starting_height();
let stores_starting_height = stores.starting_height();
let starting_height = vecs_starting_height.min(stores_starting_height);
let range = u32::from( let range = u32::from(
starting_height starting_height
@@ -208,18 +210,18 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
} }
pub fn starting_index<I, T>( pub fn starting_index<I, T>(
height_to_index: &IndexedVec<Height, I>, height_to_index: &StampedVec<Height, I>,
index_to_else: &IndexedVec<I, T>, index_to_else: &StampedVec<I, T>,
starting_height: Height, starting_height: Height,
) -> Option<I> ) -> Option<I>
where where
I: StoredType + StoredIndex + From<usize>, I: StoredType + StoredIndex + From<usize>,
T: StoredType, T: StoredType,
{ {
let h = height_to_index.height(); let h = Height::from(u64::from(height_to_index.stamp()));
if h.is_zero() { if h.is_zero() {
None None
} else if height_to_index.height() + 1_u32 == starting_height { } else if h + 1_u32 == starting_height {
Some(I::from(index_to_else.len())) Some(I::from(index_to_else.len()))
} else { } else {
height_to_index.iter().get_inner(starting_height) height_to_index.iter().get_inner(starting_height)
+167 -56
View File
@@ -1,21 +1,20 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
#![doc = "\n## Example\n\n```rust"] #![doc = "\n## Example\n\n```rust"]
#![doc = include_str!("../examples/main.rs")] #![doc = include_str!("../examples/indexer.rs")]
#![doc = "```"] #![doc = "```"]
use std::{collections::BTreeMap, path::Path, str::FromStr, thread}; use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex,
Unit, Version, Vin, Vout, setrlimit,
};
use bitcoin::{Transaction, TxIn, TxOut}; use bitcoin::{Transaction, TxIn, TxOut};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Result, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit,
};
use brk_exit::Exit; use brk_exit::Exit;
use brk_parser::Parser; use brk_parser::Parser;
use brk_store::AnyStore; use brk_store::AnyStore;
use brk_vec::{AnyVec, VecIterator}; use brk_vecs::{AnyVec, File, PAGE_SIZE, Reader, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre}; use color_eyre::eyre::{ContextCompat, eyre};
use log::{error, info}; use log::{error, info};
use rayon::prelude::*; use rayon::prelude::*;
@@ -28,11 +27,12 @@ pub use stores::*;
pub use vecs::*; pub use vecs::*;
const SNAPSHOT_BLOCK_RANGE: usize = 1000; const SNAPSHOT_BLOCK_RANGE: usize = 1000;
const COLLISIONS_CHECKED_UP_TO: u32 = 893_000; const COLLISIONS_CHECKED_UP_TO: Height = Height::new(907_000);
const VERSION: Version = Version::ONE; const VERSION: Version = Version::ONE;
#[derive(Clone)] #[derive(Clone)]
pub struct Indexer { pub struct Indexer {
pub file: Arc<File>,
pub vecs: Vecs, pub vecs: Vecs,
pub stores: Stores, pub stores: Stores,
} }
@@ -40,9 +40,20 @@ pub struct Indexer {
impl Indexer { impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result<Self> { pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result<Self> {
setrlimit()?; setrlimit()?;
let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?);
let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?;
file.set_min_len(PAGE_SIZE * 50_000_000)?;
Ok(Self { Ok(Self {
vecs: Vecs::forced_import(&outputs_dir.join("vecs/indexed"), VERSION + Version::ZERO)?, vecs,
stores: Stores::forced_import(&outputs_dir.join("stores"), VERSION + Version::ZERO)?, stores: Stores::forced_import(
&outputs_dir.join("indexed/stores"),
VERSION + Version::ZERO,
)?,
file,
}) })
} }
@@ -53,17 +64,19 @@ impl Indexer {
exit: &Exit, exit: &Exit,
check_collisions: bool, check_collisions: bool,
) -> color_eyre::Result<Indexes> { ) -> color_eyre::Result<Indexes> {
let file = self.file.clone();
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)) let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc))
.unwrap_or_else(|_report| Indexes::default()); .unwrap_or_else(|_report| Indexes::default());
// dbg!(starting_indexes); // dbg!(starting_indexes);
// panic!(); // panic!();
exit.block(); let lock = exit.lock();
self.stores self.stores
.rollback_if_needed(&mut self.vecs, &starting_indexes)?; .rollback_if_needed(&mut self.vecs, &starting_indexes)?;
self.vecs.rollback_if_needed(&starting_indexes)?; self.vecs.rollback_if_needed(&starting_indexes)?;
exit.release(); drop(lock);
let vecs = &mut self.vecs; let vecs = &mut self.vecs;
let stores = &mut self.stores; let stores = &mut self.stores;
@@ -83,32 +96,104 @@ impl Indexer {
info!("Started indexing..."); info!("Started indexing...");
let export_if_needed = |stores: &mut Stores, let should_export = |height: Height, rem: bool| -> bool {
vecs: &mut Vecs, height != 0 && (height % SNAPSHOT_BLOCK_RANGE == 0) != rem
height: Height,
rem: bool,
exit: &Exit|
-> color_eyre::Result<()> {
if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem || exit.triggered() {
return Ok(());
}
info!("Exporting...");
exit.block();
stores.commit(height)?;
vecs.flush(height)?;
exit.release();
Ok(())
}; };
let export =
|stores: &mut Stores, vecs: &mut Vecs, height: Height, exit: &Exit| -> Result<()> {
info!("Exporting...");
let _lock = exit.lock();
let i = Instant::now();
stores.commit(height).unwrap();
info!("Commited stores in {}s", i.elapsed().as_secs());
let i = Instant::now();
vecs.flush(height)?;
info!("Flushed vecs in {}s", i.elapsed().as_secs());
let i = Instant::now();
file.flush()?;
info!("Flushed file in {}s", i.elapsed().as_secs());
Ok(())
};
let mut txindex_to_first_outputindex_reader_opt = None;
let mut p2pk65addressindex_to_p2pk65bytes_reader_opt = None;
let mut p2pk33addressindex_to_p2pk33bytes_reader_opt = None;
let mut p2pkhaddressindex_to_p2pkhbytes_reader_opt = None;
let mut p2shaddressindex_to_p2shbytes_reader_opt = None;
let mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt = None;
let mut p2wshaddressindex_to_p2wshbytes_reader_opt = None;
let mut p2traddressindex_to_p2trbytes_reader_opt = None;
let mut p2aaddressindex_to_p2abytes_reader_opt = None;
let reset_mmaps_options =
|vecs: &mut Vecs,
txindex_to_first_outputindex_reader_opt: &mut Option<Reader<'static>>,
p2pk65addressindex_to_p2pk65bytes_reader_opt: &mut Option<Reader<'static>>,
p2pk33addressindex_to_p2pk33bytes_reader_opt: &mut Option<Reader<'static>>,
p2pkhaddressindex_to_p2pkhbytes_reader_opt: &mut Option<Reader<'static>>,
p2shaddressindex_to_p2shbytes_reader_opt: &mut Option<Reader<'static>>,
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt: &mut Option<Reader<'static>>,
p2wshaddressindex_to_p2wshbytes_reader_opt: &mut Option<Reader<'static>>,
p2traddressindex_to_p2trbytes_reader_opt: &mut Option<Reader<'static>>,
p2aaddressindex_to_p2abytes_reader_opt: &mut Option<Reader<'static>>| {
txindex_to_first_outputindex_reader_opt
.replace(vecs.txindex_to_first_outputindex.create_static_reader());
p2pk65addressindex_to_p2pk65bytes_reader_opt.replace(
vecs.p2pk65addressindex_to_p2pk65bytes
.create_static_reader(),
);
p2pk33addressindex_to_p2pk33bytes_reader_opt.replace(
vecs.p2pk33addressindex_to_p2pk33bytes
.create_static_reader(),
);
p2pkhaddressindex_to_p2pkhbytes_reader_opt
.replace(vecs.p2pkhaddressindex_to_p2pkhbytes.create_static_reader());
p2shaddressindex_to_p2shbytes_reader_opt
.replace(vecs.p2shaddressindex_to_p2shbytes.create_static_reader());
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.replace(
vecs.p2wpkhaddressindex_to_p2wpkhbytes
.create_static_reader(),
);
p2wshaddressindex_to_p2wshbytes_reader_opt
.replace(vecs.p2wshaddressindex_to_p2wshbytes.create_static_reader());
p2traddressindex_to_p2trbytes_reader_opt
.replace(vecs.p2traddressindex_to_p2trbytes.create_static_reader());
p2aaddressindex_to_p2abytes_reader_opt
.replace(vecs.p2aaddressindex_to_p2abytes.create_static_reader());
};
reset_mmaps_options(
vecs,
&mut txindex_to_first_outputindex_reader_opt,
&mut p2pk65addressindex_to_p2pk65bytes_reader_opt,
&mut p2pk33addressindex_to_p2pk33bytes_reader_opt,
&mut p2pkhaddressindex_to_p2pkhbytes_reader_opt,
&mut p2shaddressindex_to_p2shbytes_reader_opt,
&mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt,
&mut p2wshaddressindex_to_p2wshbytes_reader_opt,
&mut p2traddressindex_to_p2trbytes_reader_opt,
&mut p2aaddressindex_to_p2abytes_reader_opt,
);
parser.parse(start, end).iter().try_for_each( parser.parse(start, end).iter().try_for_each(
|(height, block, blockhash)| -> color_eyre::Result<()> { |(height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {height}..."); info!("Indexing block {height}...");
idxs.height = height; idxs.height = height;
let txindex_to_first_outputindex_mmap = txindex_to_first_outputindex_reader_opt.as_ref().unwrap();
let p2pk65addressindex_to_p2pk65bytes_mmap = p2pk65addressindex_to_p2pk65bytes_reader_opt.as_ref().unwrap();
let p2pk33addressindex_to_p2pk33bytes_mmap = p2pk33addressindex_to_p2pk33bytes_reader_opt.as_ref().unwrap();
let p2pkhaddressindex_to_p2pkhbytes_mmap = p2pkhaddressindex_to_p2pkhbytes_reader_opt.as_ref().unwrap();
let p2shaddressindex_to_p2shbytes_mmap = p2shaddressindex_to_p2shbytes_reader_opt.as_ref().unwrap();
let p2wpkhaddressindex_to_p2wpkhbytes_mmap = p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.as_ref().unwrap();
let p2wshaddressindex_to_p2wshbytes_mmap = p2wshaddressindex_to_p2wshbytes_reader_opt.as_ref().unwrap();
let p2traddressindex_to_p2trbytes_mmap = p2traddressindex_to_p2trbytes_reader_opt.as_ref().unwrap();
let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_reader_opt.as_ref().unwrap();
// Used to check rapidhash collisions // Used to check rapidhash collisions
let check_collisions = check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO); let check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO ;
let blockhash = BlockHash::from(blockhash); let blockhash = BlockHash::from(blockhash);
let blockhash_prefix = BlockHashPrefix::from(&blockhash); let blockhash_prefix = BlockHashPrefix::from(&blockhash);
@@ -166,9 +251,6 @@ impl Indexer {
}); });
let input_source_vec_handle = scope.spawn(|| { let input_source_vec_handle = scope.spawn(|| {
let txindex_to_first_outputindex_mmap = vecs
.txindex_to_first_outputindex.mmap().load();
let inputs = block let inputs = block
.txdata .txdata
.iter() .iter()
@@ -211,7 +293,7 @@ impl Indexer {
let vout = Vout::from(outpoint.vout); let vout = Vout::from(outpoint.vout);
let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, &txindex_to_first_outputindex_mmap)? let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_mmap)?
.context("Expect outputindex to not be none") .context("Expect outputindex to not be none")
.inspect_err(|_| { .inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout); dbg!(outpoint.txid, prev_txindex, vout);
@@ -240,16 +322,6 @@ impl Indexer {
}) })
}); });
let p2pk65addressindex_to_p2pk65bytes_mmap = vecs
.p2pk65addressindex_to_p2pk65bytes.mmap().load();
let p2pk33addressindex_to_p2pk33bytes_mmap = vecs.p2pk33addressindex_to_p2pk33bytes.mmap().load();
let p2pkhaddressindex_to_p2pkhbytes_mmap = vecs.p2pkhaddressindex_to_p2pkhbytes.mmap().load();
let p2shaddressindex_to_p2shbytes_mmap = vecs.p2shaddressindex_to_p2shbytes.mmap().load();
let p2wpkhaddressindex_to_p2wpkhbytes_mmap = vecs.p2wpkhaddressindex_to_p2wpkhbytes.mmap().load();
let p2wshaddressindex_to_p2wshbytes_mmap = vecs.p2wshaddressindex_to_p2wshbytes.mmap().load();
let p2traddressindex_to_p2trbytes_mmap = vecs.p2traddressindex_to_p2trbytes.mmap().load();
let p2aaddressindex_to_p2abytes_mmap = vecs.p2aaddressindex_to_p2abytes.mmap().load();
let outputs = block let outputs = block
.txdata .txdata
.iter() .iter()
@@ -307,35 +379,35 @@ impl Indexer {
let prev_addressbytes_opt = match outputtype { let prev_addressbytes_opt = match outputtype {
OutputType::P2PK65 => vecs OutputType::P2PK65 => vecs
.p2pk65addressindex_to_p2pk65bytes .p2pk65addressindex_to_p2pk65bytes
.get_or_read(typeindex.into(), &p2pk65addressindex_to_p2pk65bytes_mmap)? .get_or_read(typeindex.into(), p2pk65addressindex_to_p2pk65bytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2PK33 => vecs OutputType::P2PK33 => vecs
.p2pk33addressindex_to_p2pk33bytes .p2pk33addressindex_to_p2pk33bytes
.get_or_read(typeindex.into(), &p2pk33addressindex_to_p2pk33bytes_mmap)? .get_or_read(typeindex.into(), p2pk33addressindex_to_p2pk33bytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2PKH => vecs OutputType::P2PKH => vecs
.p2pkhaddressindex_to_p2pkhbytes .p2pkhaddressindex_to_p2pkhbytes
.get_or_read(typeindex.into(), &p2pkhaddressindex_to_p2pkhbytes_mmap)? .get_or_read(typeindex.into(), p2pkhaddressindex_to_p2pkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2SH => vecs OutputType::P2SH => vecs
.p2shaddressindex_to_p2shbytes .p2shaddressindex_to_p2shbytes
.get_or_read(typeindex.into(), &p2shaddressindex_to_p2shbytes_mmap)? .get_or_read(typeindex.into(), p2shaddressindex_to_p2shbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2WPKH => vecs OutputType::P2WPKH => vecs
.p2wpkhaddressindex_to_p2wpkhbytes .p2wpkhaddressindex_to_p2wpkhbytes
.get_or_read(typeindex.into(), &p2wpkhaddressindex_to_p2wpkhbytes_mmap)? .get_or_read(typeindex.into(), p2wpkhaddressindex_to_p2wpkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2WSH => vecs OutputType::P2WSH => vecs
.p2wshaddressindex_to_p2wshbytes .p2wshaddressindex_to_p2wshbytes
.get_or_read(typeindex.into(), &p2wshaddressindex_to_p2wshbytes_mmap)? .get_or_read(typeindex.into(), p2wshaddressindex_to_p2wshbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2TR => vecs OutputType::P2TR => vecs
.p2traddressindex_to_p2trbytes .p2traddressindex_to_p2trbytes
.get_or_read(typeindex.into(), &p2traddressindex_to_p2trbytes_mmap)? .get_or_read(typeindex.into(), p2traddressindex_to_p2trbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2A => vecs OutputType::P2A => vecs
.p2aaddressindex_to_p2abytes .p2aaddressindex_to_p2abytes
.get_or_read(typeindex.into(), &p2aaddressindex_to_p2abytes_mmap)? .get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())), .map(|v| AddressBytes::from(v.into_owned())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => { OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!() unreachable!()
@@ -661,6 +733,8 @@ impl Indexer {
}, },
)?; )?;
drop(txindex_to_txid_iter);
txindex_to_tx_and_txid txindex_to_tx_and_txid
.into_iter() .into_iter()
.try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> { .try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
@@ -677,15 +751,52 @@ impl Indexer {
idxs.inputindex += InputIndex::from(inputs_len); idxs.inputindex += InputIndex::from(inputs_len);
idxs.outputindex += OutputIndex::from(outputs_len); idxs.outputindex += OutputIndex::from(outputs_len);
export_if_needed(stores, vecs, height, false, exit)?;
if should_export(height, false) {
txindex_to_first_outputindex_reader_opt.take();
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
p2shaddressindex_to_p2shbytes_reader_opt.take();
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
p2traddressindex_to_p2trbytes_reader_opt.take();
p2aaddressindex_to_p2abytes_reader_opt.take();
export(stores, vecs, height, exit)?;
reset_mmaps_options(
vecs,
&mut txindex_to_first_outputindex_reader_opt,
&mut p2pk65addressindex_to_p2pk65bytes_reader_opt,
&mut p2pk33addressindex_to_p2pk33bytes_reader_opt,
&mut p2pkhaddressindex_to_p2pkhbytes_reader_opt,
&mut p2shaddressindex_to_p2shbytes_reader_opt,
&mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt,
&mut p2wshaddressindex_to_p2wshbytes_reader_opt,
&mut p2traddressindex_to_p2trbytes_reader_opt,
&mut p2aaddressindex_to_p2abytes_reader_opt,
);
}
Ok(()) Ok(())
}, },
)?; )?;
export_if_needed(stores, vecs, idxs.height, true, exit)?; if should_export(idxs.height, true) {
txindex_to_first_outputindex_reader_opt.take();
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
p2shaddressindex_to_p2shbytes_reader_opt.take();
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
p2traddressindex_to_p2trbytes_reader_opt.take();
p2aaddressindex_to_p2abytes_reader_opt.take();
export(stores, vecs, idxs.height, exit)?;
}
stores.rotate_memtables(); let i = Instant::now();
file.punch_holes()?;
info!("Punched holes in file in {}s", i.elapsed().as_secs());
Ok(starting_indexes) Ok(starting_indexes)
} }
+9 -15
View File
@@ -5,7 +5,7 @@ use brk_core::{
OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
}; };
use brk_store::{AnyStore, Store}; use brk_store::{AnyStore, Store};
use brk_vec::{AnyIterableVec, VecIterator}; use brk_vecs::{AnyIterableVec, VecIterator};
use fjall::{PersistMode, TransactionalKeyspace}; use fjall::{PersistMode, TransactionalKeyspace};
use rayon::prelude::*; use rayon::prelude::*;
@@ -72,7 +72,7 @@ impl Stores {
path, path,
"p2aaddressindex_with_outputindex", "p2aaddressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
let p2pk33addressindex_with_outputindex = scope.spawn(|| { let p2pk33addressindex_with_outputindex = scope.spawn(|| {
@@ -81,7 +81,7 @@ impl Stores {
path, path,
"p2pk33addressindex_with_outputindex", "p2pk33addressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
let p2pk65addressindex_with_outputindex = scope.spawn(|| { let p2pk65addressindex_with_outputindex = scope.spawn(|| {
@@ -90,7 +90,7 @@ impl Stores {
path, path,
"p2pk65addressindex_with_outputindex", "p2pk65addressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
let p2pkhaddressindex_with_outputindex = scope.spawn(|| { let p2pkhaddressindex_with_outputindex = scope.spawn(|| {
@@ -99,7 +99,7 @@ impl Stores {
path, path,
"p2pkhaddressindex_with_outputindex", "p2pkhaddressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
let p2shaddressindex_with_outputindex = scope.spawn(|| { let p2shaddressindex_with_outputindex = scope.spawn(|| {
@@ -108,7 +108,7 @@ impl Stores {
path, path,
"p2shaddressindex_with_outputindex", "p2shaddressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
let p2traddressindex_with_outputindex = scope.spawn(|| { let p2traddressindex_with_outputindex = scope.spawn(|| {
@@ -117,7 +117,7 @@ impl Stores {
path, path,
"p2traddressindex_with_outputindex", "p2traddressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
let p2wpkhaddressindex_with_outputindex = scope.spawn(|| { let p2wpkhaddressindex_with_outputindex = scope.spawn(|| {
@@ -126,7 +126,7 @@ impl Stores {
path, path,
"p2wpkhaddressindex_with_outputindex", "p2wpkhaddressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
let p2wshaddressindex_with_outputindex = scope.spawn(|| { let p2wshaddressindex_with_outputindex = scope.spawn(|| {
@@ -135,7 +135,7 @@ impl Stores {
path, path,
"p2wshaddressindex_with_outputindex", "p2wshaddressindex_with_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Some(None), Some(false),
) )
}); });
@@ -181,12 +181,6 @@ impl Stores {
.map_err(|e| e.into()) .map_err(|e| e.into())
} }
pub fn rotate_memtables(&self) {
self.as_slice()
.into_iter()
.for_each(|store| store.rotate_memtable());
}
fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 11] { fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 11] {
[ [
&self.addressbyteshash_to_typeindex, &self.addressbyteshash_to_typeindex,
+184 -186
View File
@@ -1,4 +1,4 @@
use std::path::Path; use std::sync::Arc;
use brk_core::{ use brk_core::{
AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex, AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
@@ -8,7 +8,7 @@ use brk_core::{
RawLockTime, Result, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, RawLockTime, Result, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion,
Txid, TypeIndex, UnknownOutputIndex, Version, Weight, Txid, TypeIndex, UnknownOutputIndex, Version, Weight,
}; };
use brk_vec::{AnyCollectableVec, AnyIndexedVec, Format, IndexedVec}; use brk_vecs::{AnyCollectableVec, AnyStampedVec, File, Format, StampedVec};
use rayon::prelude::*; use rayon::prelude::*;
use crate::Indexes; use crate::Indexes;
@@ -17,317 +17,317 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)] #[derive(Clone)]
pub struct Vecs { pub struct Vecs {
pub emptyoutputindex_to_txindex: IndexedVec<EmptyOutputIndex, TxIndex>, pub emptyoutputindex_to_txindex: StampedVec<EmptyOutputIndex, TxIndex>,
pub height_to_blockhash: IndexedVec<Height, BlockHash>, pub height_to_blockhash: StampedVec<Height, BlockHash>,
pub height_to_difficulty: IndexedVec<Height, StoredF64>, pub height_to_difficulty: StampedVec<Height, StoredF64>,
pub height_to_first_emptyoutputindex: IndexedVec<Height, EmptyOutputIndex>, pub height_to_first_emptyoutputindex: StampedVec<Height, EmptyOutputIndex>,
pub height_to_first_inputindex: IndexedVec<Height, InputIndex>, pub height_to_first_inputindex: StampedVec<Height, InputIndex>,
pub height_to_first_opreturnindex: IndexedVec<Height, OpReturnIndex>, pub height_to_first_opreturnindex: StampedVec<Height, OpReturnIndex>,
pub height_to_first_outputindex: IndexedVec<Height, OutputIndex>, pub height_to_first_outputindex: StampedVec<Height, OutputIndex>,
pub height_to_first_p2aaddressindex: IndexedVec<Height, P2AAddressIndex>, pub height_to_first_p2aaddressindex: StampedVec<Height, P2AAddressIndex>,
pub height_to_first_p2msoutputindex: IndexedVec<Height, P2MSOutputIndex>, pub height_to_first_p2msoutputindex: StampedVec<Height, P2MSOutputIndex>,
pub height_to_first_p2pk33addressindex: IndexedVec<Height, P2PK33AddressIndex>, pub height_to_first_p2pk33addressindex: StampedVec<Height, P2PK33AddressIndex>,
pub height_to_first_p2pk65addressindex: IndexedVec<Height, P2PK65AddressIndex>, pub height_to_first_p2pk65addressindex: StampedVec<Height, P2PK65AddressIndex>,
pub height_to_first_p2pkhaddressindex: IndexedVec<Height, P2PKHAddressIndex>, pub height_to_first_p2pkhaddressindex: StampedVec<Height, P2PKHAddressIndex>,
pub height_to_first_p2shaddressindex: IndexedVec<Height, P2SHAddressIndex>, pub height_to_first_p2shaddressindex: StampedVec<Height, P2SHAddressIndex>,
pub height_to_first_p2traddressindex: IndexedVec<Height, P2TRAddressIndex>, pub height_to_first_p2traddressindex: StampedVec<Height, P2TRAddressIndex>,
pub height_to_first_p2wpkhaddressindex: IndexedVec<Height, P2WPKHAddressIndex>, pub height_to_first_p2wpkhaddressindex: StampedVec<Height, P2WPKHAddressIndex>,
pub height_to_first_p2wshaddressindex: IndexedVec<Height, P2WSHAddressIndex>, pub height_to_first_p2wshaddressindex: StampedVec<Height, P2WSHAddressIndex>,
pub height_to_first_txindex: IndexedVec<Height, TxIndex>, pub height_to_first_txindex: StampedVec<Height, TxIndex>,
pub height_to_first_unknownoutputindex: IndexedVec<Height, UnknownOutputIndex>, pub height_to_first_unknownoutputindex: StampedVec<Height, UnknownOutputIndex>,
/// Doesn't guarantee continuity due to possible reorgs /// Doesn't guarantee continuity due to possible reorgs
pub height_to_timestamp: IndexedVec<Height, Timestamp>, pub height_to_timestamp: StampedVec<Height, Timestamp>,
pub height_to_total_size: IndexedVec<Height, StoredUsize>, pub height_to_total_size: StampedVec<Height, StoredUsize>,
pub height_to_weight: IndexedVec<Height, Weight>, pub height_to_weight: StampedVec<Height, Weight>,
/// If outputindex == Outputindex::MAX then it's coinbase /// If outputindex == Outputindex::MAX then it's coinbase
pub inputindex_to_outputindex: IndexedVec<InputIndex, OutputIndex>, pub inputindex_to_outputindex: StampedVec<InputIndex, OutputIndex>,
pub opreturnindex_to_txindex: IndexedVec<OpReturnIndex, TxIndex>, pub opreturnindex_to_txindex: StampedVec<OpReturnIndex, TxIndex>,
pub outputindex_to_outputtype: IndexedVec<OutputIndex, OutputType>, pub outputindex_to_outputtype: StampedVec<OutputIndex, OutputType>,
pub outputindex_to_typeindex: IndexedVec<OutputIndex, TypeIndex>, pub outputindex_to_typeindex: StampedVec<OutputIndex, TypeIndex>,
pub outputindex_to_value: IndexedVec<OutputIndex, Sats>, pub outputindex_to_value: StampedVec<OutputIndex, Sats>,
pub p2aaddressindex_to_p2abytes: IndexedVec<P2AAddressIndex, P2ABytes>, pub p2aaddressindex_to_p2abytes: StampedVec<P2AAddressIndex, P2ABytes>,
pub p2msoutputindex_to_txindex: IndexedVec<P2MSOutputIndex, TxIndex>, pub p2msoutputindex_to_txindex: StampedVec<P2MSOutputIndex, TxIndex>,
pub p2pk33addressindex_to_p2pk33bytes: IndexedVec<P2PK33AddressIndex, P2PK33Bytes>, pub p2pk33addressindex_to_p2pk33bytes: StampedVec<P2PK33AddressIndex, P2PK33Bytes>,
pub p2pk65addressindex_to_p2pk65bytes: IndexedVec<P2PK65AddressIndex, P2PK65Bytes>, pub p2pk65addressindex_to_p2pk65bytes: StampedVec<P2PK65AddressIndex, P2PK65Bytes>,
pub p2pkhaddressindex_to_p2pkhbytes: IndexedVec<P2PKHAddressIndex, P2PKHBytes>, pub p2pkhaddressindex_to_p2pkhbytes: StampedVec<P2PKHAddressIndex, P2PKHBytes>,
pub p2shaddressindex_to_p2shbytes: IndexedVec<P2SHAddressIndex, P2SHBytes>, pub p2shaddressindex_to_p2shbytes: StampedVec<P2SHAddressIndex, P2SHBytes>,
pub p2traddressindex_to_p2trbytes: IndexedVec<P2TRAddressIndex, P2TRBytes>, pub p2traddressindex_to_p2trbytes: StampedVec<P2TRAddressIndex, P2TRBytes>,
pub p2wpkhaddressindex_to_p2wpkhbytes: IndexedVec<P2WPKHAddressIndex, P2WPKHBytes>, pub p2wpkhaddressindex_to_p2wpkhbytes: StampedVec<P2WPKHAddressIndex, P2WPKHBytes>,
pub p2wshaddressindex_to_p2wshbytes: IndexedVec<P2WSHAddressIndex, P2WSHBytes>, pub p2wshaddressindex_to_p2wshbytes: StampedVec<P2WSHAddressIndex, P2WSHBytes>,
pub txindex_to_base_size: IndexedVec<TxIndex, StoredU32>, pub txindex_to_base_size: StampedVec<TxIndex, StoredU32>,
pub txindex_to_first_inputindex: IndexedVec<TxIndex, InputIndex>, pub txindex_to_first_inputindex: StampedVec<TxIndex, InputIndex>,
pub txindex_to_first_outputindex: IndexedVec<TxIndex, OutputIndex>, pub txindex_to_first_outputindex: StampedVec<TxIndex, OutputIndex>,
pub txindex_to_is_explicitly_rbf: IndexedVec<TxIndex, bool>, pub txindex_to_is_explicitly_rbf: StampedVec<TxIndex, bool>,
pub txindex_to_rawlocktime: IndexedVec<TxIndex, RawLockTime>, pub txindex_to_rawlocktime: StampedVec<TxIndex, RawLockTime>,
pub txindex_to_total_size: IndexedVec<TxIndex, StoredU32>, pub txindex_to_total_size: StampedVec<TxIndex, StoredU32>,
pub txindex_to_txid: IndexedVec<TxIndex, Txid>, pub txindex_to_txid: StampedVec<TxIndex, Txid>,
pub txindex_to_txversion: IndexedVec<TxIndex, TxVersion>, pub txindex_to_txversion: StampedVec<TxIndex, TxVersion>,
pub unknownoutputindex_to_txindex: IndexedVec<UnknownOutputIndex, TxIndex>, pub unknownoutputindex_to_txindex: StampedVec<UnknownOutputIndex, TxIndex>,
} }
impl Vecs { impl Vecs {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> { pub fn forced_import(file: &Arc<File>, version: Version) -> color_eyre::Result<Self> {
Ok(Self { Ok(Self {
emptyoutputindex_to_txindex: IndexedVec::forced_import( emptyoutputindex_to_txindex: StampedVec::forced_import(
path, file,
"txindex", "txindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_blockhash: IndexedVec::forced_import( height_to_blockhash: StampedVec::forced_import(
path, file,
"blockhash", "blockhash",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_difficulty: IndexedVec::forced_import( height_to_difficulty: StampedVec::forced_import(
path, file,
"difficulty", "difficulty",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_emptyoutputindex: IndexedVec::forced_import( height_to_first_emptyoutputindex: StampedVec::forced_import(
path, file,
"first_emptyoutputindex", "first_emptyoutputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_inputindex: IndexedVec::forced_import( height_to_first_inputindex: StampedVec::forced_import(
path, file,
"first_inputindex", "first_inputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_opreturnindex: IndexedVec::forced_import( height_to_first_opreturnindex: StampedVec::forced_import(
path, file,
"first_opreturnindex", "first_opreturnindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_outputindex: IndexedVec::forced_import( height_to_first_outputindex: StampedVec::forced_import(
path, file,
"first_outputindex", "first_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2aaddressindex: IndexedVec::forced_import( height_to_first_p2aaddressindex: StampedVec::forced_import(
path, file,
"first_p2aaddressindex", "first_p2aaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2msoutputindex: IndexedVec::forced_import( height_to_first_p2msoutputindex: StampedVec::forced_import(
path, file,
"first_p2msoutputindex", "first_p2msoutputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2pk33addressindex: IndexedVec::forced_import( height_to_first_p2pk33addressindex: StampedVec::forced_import(
path, file,
"first_p2pk33addressindex", "first_p2pk33addressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2pk65addressindex: IndexedVec::forced_import( height_to_first_p2pk65addressindex: StampedVec::forced_import(
path, file,
"first_p2pk65addressindex", "first_p2pk65addressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2pkhaddressindex: IndexedVec::forced_import( height_to_first_p2pkhaddressindex: StampedVec::forced_import(
path, file,
"first_p2pkhaddressindex", "first_p2pkhaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2shaddressindex: IndexedVec::forced_import( height_to_first_p2shaddressindex: StampedVec::forced_import(
path, file,
"first_p2shaddressindex", "first_p2shaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2traddressindex: IndexedVec::forced_import( height_to_first_p2traddressindex: StampedVec::forced_import(
path, file,
"first_p2traddressindex", "first_p2traddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2wpkhaddressindex: IndexedVec::forced_import( height_to_first_p2wpkhaddressindex: StampedVec::forced_import(
path, file,
"first_p2wpkhaddressindex", "first_p2wpkhaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_p2wshaddressindex: IndexedVec::forced_import( height_to_first_p2wshaddressindex: StampedVec::forced_import(
path, file,
"first_p2wshaddressindex", "first_p2wshaddressindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_txindex: IndexedVec::forced_import( height_to_first_txindex: StampedVec::forced_import(
path, file,
"first_txindex", "first_txindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_first_unknownoutputindex: IndexedVec::forced_import( height_to_first_unknownoutputindex: StampedVec::forced_import(
path, file,
"first_unknownoutputindex", "first_unknownoutputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_timestamp: IndexedVec::forced_import( height_to_timestamp: StampedVec::forced_import(
path, file,
"timestamp", "timestamp",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_total_size: IndexedVec::forced_import( height_to_total_size: StampedVec::forced_import(
path, file,
"total_size", "total_size",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
height_to_weight: IndexedVec::forced_import( height_to_weight: StampedVec::forced_import(
path, file,
"weight", "weight",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
inputindex_to_outputindex: IndexedVec::forced_import( inputindex_to_outputindex: StampedVec::forced_import(
path, file,
"outputindex", "outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
opreturnindex_to_txindex: IndexedVec::forced_import( opreturnindex_to_txindex: StampedVec::forced_import(
path, file,
"txindex", "txindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
outputindex_to_outputtype: IndexedVec::forced_import( outputindex_to_outputtype: StampedVec::forced_import(
path, file,
"outputtype", "outputtype",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
outputindex_to_typeindex: IndexedVec::forced_import( outputindex_to_typeindex: StampedVec::forced_import(
path, file,
"typeindex", "typeindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
outputindex_to_value: IndexedVec::forced_import( outputindex_to_value: StampedVec::forced_import(
path, file,
"value", "value",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2aaddressindex_to_p2abytes: IndexedVec::forced_import( p2aaddressindex_to_p2abytes: StampedVec::forced_import(
path, file,
"p2abytes", "p2abytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2msoutputindex_to_txindex: IndexedVec::forced_import( p2msoutputindex_to_txindex: StampedVec::forced_import(
path, file,
"txindex", "txindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2pk33addressindex_to_p2pk33bytes: IndexedVec::forced_import( p2pk33addressindex_to_p2pk33bytes: StampedVec::forced_import(
path, file,
"p2pk33bytes", "p2pk33bytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2pk65addressindex_to_p2pk65bytes: IndexedVec::forced_import( p2pk65addressindex_to_p2pk65bytes: StampedVec::forced_import(
path, file,
"p2pk65bytes", "p2pk65bytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2pkhaddressindex_to_p2pkhbytes: IndexedVec::forced_import( p2pkhaddressindex_to_p2pkhbytes: StampedVec::forced_import(
path, file,
"p2pkhbytes", "p2pkhbytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2shaddressindex_to_p2shbytes: IndexedVec::forced_import( p2shaddressindex_to_p2shbytes: StampedVec::forced_import(
path, file,
"p2shbytes", "p2shbytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2traddressindex_to_p2trbytes: IndexedVec::forced_import( p2traddressindex_to_p2trbytes: StampedVec::forced_import(
path, file,
"p2trbytes", "p2trbytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2wpkhaddressindex_to_p2wpkhbytes: IndexedVec::forced_import( p2wpkhaddressindex_to_p2wpkhbytes: StampedVec::forced_import(
path, file,
"p2wpkhbytes", "p2wpkhbytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
p2wshaddressindex_to_p2wshbytes: IndexedVec::forced_import( p2wshaddressindex_to_p2wshbytes: StampedVec::forced_import(
path, file,
"p2wshbytes", "p2wshbytes",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_base_size: IndexedVec::forced_import( txindex_to_base_size: StampedVec::forced_import(
path, file,
"base_size", "base_size",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_first_inputindex: IndexedVec::forced_import( txindex_to_first_inputindex: StampedVec::forced_import(
path, file,
"first_inputindex", "first_inputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_first_outputindex: IndexedVec::forced_import( txindex_to_first_outputindex: StampedVec::forced_import(
path, file,
"first_outputindex", "first_outputindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_is_explicitly_rbf: IndexedVec::forced_import( txindex_to_is_explicitly_rbf: StampedVec::forced_import(
path, file,
"is_explicitly_rbf", "is_explicitly_rbf",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_rawlocktime: IndexedVec::forced_import( txindex_to_rawlocktime: StampedVec::forced_import(
path, file,
"rawlocktime", "rawlocktime",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_total_size: IndexedVec::forced_import( txindex_to_total_size: StampedVec::forced_import(
path, file,
"total_size", "total_size",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_txid: IndexedVec::forced_import( txindex_to_txid: StampedVec::forced_import(
path, file,
"txid", "txid",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
txindex_to_txversion: IndexedVec::forced_import( txindex_to_txversion: StampedVec::forced_import(
path, file,
"txversion", "txversion",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
)?, )?,
unknownoutputindex_to_txindex: IndexedVec::forced_import( unknownoutputindex_to_txindex: StampedVec::forced_import(
path, file,
"txindex", "txindex",
version + VERSION + Version::ZERO, version + VERSION + Version::ZERO,
Format::Raw, Format::Raw,
@@ -357,94 +357,92 @@ impl Vecs {
unknownoutputindex, unknownoutputindex,
} = starting_indexes; } = starting_indexes;
let stamp = u64::from(saved_height).into();
self.emptyoutputindex_to_txindex self.emptyoutputindex_to_txindex
.truncate_if_needed(emptyoutputindex, saved_height)?; .truncate_if_needed(emptyoutputindex, stamp)?;
self.height_to_blockhash self.height_to_blockhash.truncate_if_needed(height, stamp)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty self.height_to_difficulty
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_emptyoutputindex self.height_to_first_emptyoutputindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_inputindex self.height_to_first_inputindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_opreturnindex self.height_to_first_opreturnindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_outputindex self.height_to_first_outputindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2aaddressindex self.height_to_first_p2aaddressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2msoutputindex self.height_to_first_p2msoutputindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2pk33addressindex self.height_to_first_p2pk33addressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2pk65addressindex self.height_to_first_p2pk65addressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2pkhaddressindex self.height_to_first_p2pkhaddressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2shaddressindex self.height_to_first_p2shaddressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2traddressindex self.height_to_first_p2traddressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2wpkhaddressindex self.height_to_first_p2wpkhaddressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_p2wshaddressindex self.height_to_first_p2wshaddressindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_txindex self.height_to_first_txindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_first_unknownoutputindex self.height_to_first_unknownoutputindex
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_timestamp self.height_to_timestamp.truncate_if_needed(height, stamp)?;
.truncate_if_needed(height, saved_height)?;
self.height_to_total_size self.height_to_total_size
.truncate_if_needed(height, saved_height)?; .truncate_if_needed(height, stamp)?;
self.height_to_weight self.height_to_weight.truncate_if_needed(height, stamp)?;
.truncate_if_needed(height, saved_height)?;
self.inputindex_to_outputindex self.inputindex_to_outputindex
.truncate_if_needed(inputindex, saved_height)?; .truncate_if_needed(inputindex, stamp)?;
self.opreturnindex_to_txindex self.opreturnindex_to_txindex
.truncate_if_needed(opreturnindex, saved_height)?; .truncate_if_needed(opreturnindex, stamp)?;
self.outputindex_to_outputtype self.outputindex_to_outputtype
.truncate_if_needed(outputindex, saved_height)?; .truncate_if_needed(outputindex, stamp)?;
self.outputindex_to_typeindex self.outputindex_to_typeindex
.truncate_if_needed(outputindex, saved_height)?; .truncate_if_needed(outputindex, stamp)?;
self.outputindex_to_value self.outputindex_to_value
.truncate_if_needed(outputindex, saved_height)?; .truncate_if_needed(outputindex, stamp)?;
self.p2aaddressindex_to_p2abytes self.p2aaddressindex_to_p2abytes
.truncate_if_needed(p2aaddressindex, saved_height)?; .truncate_if_needed(p2aaddressindex, stamp)?;
self.p2msoutputindex_to_txindex self.p2msoutputindex_to_txindex
.truncate_if_needed(p2msoutputindex, saved_height)?; .truncate_if_needed(p2msoutputindex, stamp)?;
self.p2pk33addressindex_to_p2pk33bytes self.p2pk33addressindex_to_p2pk33bytes
.truncate_if_needed(p2pk33addressindex, saved_height)?; .truncate_if_needed(p2pk33addressindex, stamp)?;
self.p2pk65addressindex_to_p2pk65bytes self.p2pk65addressindex_to_p2pk65bytes
.truncate_if_needed(p2pk65addressindex, saved_height)?; .truncate_if_needed(p2pk65addressindex, stamp)?;
self.p2pkhaddressindex_to_p2pkhbytes self.p2pkhaddressindex_to_p2pkhbytes
.truncate_if_needed(p2pkhaddressindex, saved_height)?; .truncate_if_needed(p2pkhaddressindex, stamp)?;
self.p2shaddressindex_to_p2shbytes self.p2shaddressindex_to_p2shbytes
.truncate_if_needed(p2shaddressindex, saved_height)?; .truncate_if_needed(p2shaddressindex, stamp)?;
self.p2traddressindex_to_p2trbytes self.p2traddressindex_to_p2trbytes
.truncate_if_needed(p2traddressindex, saved_height)?; .truncate_if_needed(p2traddressindex, stamp)?;
self.p2wpkhaddressindex_to_p2wpkhbytes self.p2wpkhaddressindex_to_p2wpkhbytes
.truncate_if_needed(p2wpkhaddressindex, saved_height)?; .truncate_if_needed(p2wpkhaddressindex, stamp)?;
self.p2wshaddressindex_to_p2wshbytes self.p2wshaddressindex_to_p2wshbytes
.truncate_if_needed(p2wshaddressindex, saved_height)?; .truncate_if_needed(p2wshaddressindex, stamp)?;
self.txindex_to_base_size self.txindex_to_base_size
.truncate_if_needed(txindex, saved_height)?; .truncate_if_needed(txindex, stamp)?;
self.txindex_to_first_inputindex self.txindex_to_first_inputindex
.truncate_if_needed(txindex, saved_height)?; .truncate_if_needed(txindex, stamp)?;
self.txindex_to_first_outputindex self.txindex_to_first_outputindex
.truncate_if_needed(txindex, saved_height)?; .truncate_if_needed(txindex, stamp)?;
self.txindex_to_is_explicitly_rbf self.txindex_to_is_explicitly_rbf
.truncate_if_needed(txindex, saved_height)?; .truncate_if_needed(txindex, stamp)?;
self.txindex_to_rawlocktime self.txindex_to_rawlocktime
.truncate_if_needed(txindex, saved_height)?; .truncate_if_needed(txindex, stamp)?;
self.txindex_to_total_size self.txindex_to_total_size
.truncate_if_needed(txindex, saved_height)?; .truncate_if_needed(txindex, stamp)?;
self.txindex_to_txid self.txindex_to_txid.truncate_if_needed(txindex, stamp)?;
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txversion self.txindex_to_txversion
.truncate_if_needed(txindex, saved_height)?; .truncate_if_needed(txindex, stamp)?;
self.unknownoutputindex_to_txindex self.unknownoutputindex_to_txindex
.truncate_if_needed(unknownoutputindex, saved_height)?; .truncate_if_needed(unknownoutputindex, stamp)?;
Ok(()) Ok(())
} }
@@ -481,14 +479,14 @@ impl Vecs {
pub fn flush(&mut self, height: Height) -> Result<()> { pub fn flush(&mut self, height: Height) -> Result<()> {
self.mut_vecs() self.mut_vecs()
.into_par_iter() .into_par_iter()
.try_for_each(|vec| vec.flush(height)) .try_for_each(|vec| vec.flush(u64::from(height).into()))
} }
pub fn starting_height(&mut self) -> Height { pub fn starting_height(&mut self) -> Height {
self.mut_vecs() self.mut_vecs()
.into_iter() .into_iter()
.map(|vec| { .map(|vec| {
let h = vec.height(); let h = Height::from(u64::from(vec.stamp()));
if h > Height::ZERO { h.incremented() } else { h } if h > Height::ZERO { h.incremented() } else { h }
}) })
.min() .min()
@@ -544,7 +542,7 @@ impl Vecs {
] ]
} }
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> { fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStampedVec> {
vec![ vec![
&mut self.emptyoutputindex_to_txindex, &mut self.emptyoutputindex_to_txindex,
&mut self.height_to_blockhash, &mut self.height_to_blockhash,
+2 -2
View File
@@ -11,10 +11,10 @@ repository.workspace = true
brk_core = { workspace = true } brk_core = { workspace = true }
brk_computer = { workspace = true } brk_computer = { workspace = true }
brk_indexer = { workspace = true } brk_indexer = { workspace = true }
brk_vec = { workspace = true } brk_vecs = { workspace = true }
color-eyre = { workspace = true } color-eyre = { workspace = true }
derive_deref = { workspace = true } derive_deref = { workspace = true }
brk_rmcp = { workspace = true } rmcp = { workspace = true }
schemars = { workspace = true } schemars = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
+1 -1
View File
@@ -3,7 +3,7 @@ use std::path::Path;
use brk_computer::Computer; use brk_computer::Computer;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_interface::{Index, Interface, Params, ParamsOpt}; use brk_interface::{Index, Interface, Params, ParamsOpt};
use brk_vec::{Computation, Format}; use brk_vecs::{Computation, Format};
pub fn main() -> color_eyre::Result<()> { pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?; color_eyre::install()?;
+1 -1
View File
@@ -1,5 +1,5 @@
use brk_rmcp::schemars::JsonSchema;
use color_eyre::eyre::eyre; use color_eyre::eyre::eyre;
use rmcp::schemars::JsonSchema;
use serde::Deserialize; use serde::Deserialize;
#[allow(clippy::upper_case_acronyms)] #[allow(clippy::upper_case_acronyms)]
+2 -2
View File
@@ -8,7 +8,7 @@ use std::collections::BTreeMap;
use brk_computer::Computer; use brk_computer::Computer;
use brk_core::{Height, Result}; use brk_core::{Height, Result};
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIndexedVec}; use brk_vecs::{AnyCollectableVec, AnyStampedVec};
use tabled::settings::Style; use tabled::settings::Style;
mod deser; mod deser;
@@ -47,7 +47,7 @@ impl<'a> Interface<'a> {
} }
pub fn get_height(&self) -> Height { pub fn get_height(&self) -> Height {
self._indexer.vecs.height_to_blockhash.height() Height::from(u64::from(self._indexer.vecs.height_to_blockhash.stamp()))
} }
pub fn search(&self, params: &Params) -> Vec<(String, &&dyn AnyCollectableVec)> { pub fn search(&self, params: &Params) -> Vec<(String, &&dyn AnyCollectableVec)> {
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::Deref; use std::ops::Deref;
use brk_rmcp::schemars::{self, JsonSchema}; use rmcp::schemars::{self, JsonSchema};
use serde::Deserialize; use serde::Deserialize;
use crate::{ use crate::{
+1 -28
View File
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use brk_computer::Computer; use brk_computer::Computer;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_vec::AnyCollectableVec; use brk_vecs::AnyCollectableVec;
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use crate::pagination::{PaginatedIndexParam, PaginationParam}; use crate::pagination::{PaginatedIndexParam, PaginationParam};
@@ -101,33 +101,6 @@ impl<'a> Vecs<'a> {
fn insert(&mut self, vec: &'a dyn AnyCollectableVec) { fn insert(&mut self, vec: &'a dyn AnyCollectableVec) {
let name = vec.name(); let name = vec.name();
let serialized_index = vec.index_type_to_string(); let serialized_index = vec.index_type_to_string();
// let split = name.split("_to_").collect::<Vec<_>>();
// if split.len() != 1
// && !(split.len() == 2
// && split.first().is_some_and(|s| {
// s == &"up"
// || s == &"start"
// || s.ends_with("relative")
// || s.starts_with("from")
// || s == &"cumulative_up"
// || s.starts_with("cumulative_start")
// || s.starts_with("cumulative_from")
// || s == &"activity"
// }))
// && !(split.len() == 3
// && split.first().is_some_and(|s| {
// s == &"up"
// || s == &"start"
// || s.starts_with("from")
// || s == &"cumulative_up"
// || s == &"cumulative_start"
// || s.starts_with("cumulative_from")
// })
// && split.get(1).is_some_and(|s| s.ends_with("relative")))
// {
// dbg!((&serialized_index, &name, &split));
// unreachable!();
// }
let index = Index::try_from(serialized_index) let index = Index::try_from(serialized_index)
.inspect_err(|_| { .inspect_err(|_| {
dbg!(&serialized_index); dbg!(&serialized_index);
+1 -1
View File
@@ -26,7 +26,7 @@ pub fn init(path: Option<&Path>) {
}); });
Builder::from_env(Env::default().default_filter_or( Builder::from_env(Env::default().default_filter_or(
"info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off,rmcp=off,brk_rmcp=off,tracing=off", "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off,rmcp=off,rmcp=off,tracing=off",
)) ))
.format(move |buf, record| { .format(move |buf, record| {
let date_time = Timestamp::now() let date_time = Timestamp::now()

Some files were not shown because too many files have changed in this diff Show More