Compare commits

...

24 Commits

Author SHA1 Message Date
nym21 6fb6abcbe5 release: v0.0.44 2025-06-07 18:53:51 +02:00
nym21 dc449dafd1 websites: default: up deps + fix css 2025-06-07 18:53:34 +02:00
nym21 ecdaeebbfb release: v0.0.43 2025-06-07 13:48:25 +02:00
nym21 fa958b59bd fetcher: support new api 2025-06-07 13:48:07 +02:00
nym21 fb3d8521cd computer: coinblocks fix overflow 2025-06-07 13:29:08 +02:00
nym21 608c401cf3 release: v0.0.42 2025-06-07 10:40:32 +02:00
nym21 1c3da90a24 release: v0.0.41 2025-06-07 10:31:36 +02:00
nym21 34567f3375 changelog: reset last 2025-06-07 10:31:07 +02:00
nym21 51bcbeb48f global: multiple fixes 2025-06-07 09:30:42 +02:00
nym21 cc0f9c42df global: snapshot 2025-06-06 16:08:20 +02:00
nym21 a11bf5523b global: wip 2025-06-06 12:23:45 +02:00
nym21 1921c3d901 global: wip 2025-06-06 10:46:38 +02:00
nym21 d568469e8b global: works but data is wrong 2025-06-04 17:01:16 +02:00
nym21 20d5c7e8d5 global: wip + fixed eager mode 2025-06-03 17:49:20 +02:00
nym21 9f289ed9de global: wip 2025-06-03 10:11:51 +02:00
nym21 93ee5e480b global: wip 2025-06-02 18:22:42 +02:00
nym21 98a312701f computer: more frequent flushes 2025-06-01 16:11:13 +02:00
nym21 cbcf603b63 global: wip 2025-06-01 14:37:19 +02:00
nym21 f976f672cf global: wip 2025-05-31 20:45:59 +02:00
nym21 cfc3081e8a global: snapshot 2025-05-29 10:39:58 +02:00
nym21 99818924ee global: snapshot 2025-05-28 16:53:18 +02:00
nym21 9bbf3a027f global: snapshot 2025-05-28 15:42:55 +02:00
nym21 93e01902e3 global: snapshot 2025-05-27 15:19:53 +02:00
nym21 34919aba05 global: versions 2025-05-26 11:34:37 +02:00
267 changed files with 25004 additions and 9033 deletions
+1 -20
View File
@@ -8,30 +8,11 @@ target
*\ copy*
# Ignored
/_*
_*
# Editors
.vscode
.zed
# Flamegraph
flamegraph/
flamegraph.svg
# Benchmarks
benches
# Snapshots
snapshots*/
# Docker
docker/kibo
# Types
paths.d.ts
# Outputs
_outputs
# Logs
.log
+2 -31
View File
@@ -3,38 +3,9 @@
![Image of the kibo Web App version 0.X.Y](https://github.com/kibo-money/kibo/blob/main/_assets/v0.X.Y.jpg)
-->
# v0.6.0 | WIP | A new beginning
# v0.X.0 | WIP | A new beginning
## Global
- Completely redesign the back-end
- Merged parser and server crates into a single project (and thus executable), so now both will run at the same time with a single `cargo run -r` [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
- Added `--no-server` and `--no-parser` to disable each if needed
- Improved executable parameters
- Started using `log` and `env_logger` crates instead of custom code [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
- Improved logs
- Fixed input being unfocused right after being focused in Brave browser [#9a9ae61](https://github.com/kibo-money/kibo/commit/9a9ae614d07b54c08b7e9c0e2aefe3b52fdb93c5)
- Reworked server's API code [#6ab0f46]( https://github.com/kibo-money/kibo/commit/6ab0f463119a902a1b7ca9691b54f61543bb8f2f)
- New route format: `/api/date-to-realized-price` is now `/api/realized-price?kind=date`
- Added status and timing to logs
- Updated website packages
- Added API support for datasets by timestamp (by merging any dataset by height with the height to timestamp dataset and so it still uses heights as chunk ids) [#ca00f3f](https://github.com/kibo-money/kibo/commit/ca00f3f71526f0c5c16021024fec7e5c6e47221c)
- `/api/realized-price?kind=t`
- `/api/realized-price?kind=timestamp&chunk=860000`
- Created separate crate for indexing called `bindex`
- Created a crate a storage engine specialized in storing datasets that have indexes as keys and thus can be represented by an array/vec called `storable-vec`
- Removed the need for the `-txindex=1` parameter when starting your Bitcoin Core node as kibo has its own indexes now
## Git
Added git tags for each version though Markdown won't display formatted on Github so left the default text
## Deprecated
Moved Sanakirja database wrapper to its own crate (`snkrj`) and added a robust auto defragmentation to improve disk usage without the need for user's intervention.
Since it's not used anymore it will moved out of the repository relatively soon.
Full rewrite
# [kibo-v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
Generated
+198 -179
View File
@@ -28,19 +28,6 @@ dependencies = [
"cpufeatures",
]
[[package]]
name = "ahash"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"getrandom 0.3.3",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
name = "aho-corasick"
version = "1.1.3"
@@ -88,9 +75,9 @@ dependencies = [
[[package]]
name = "anstream"
version = "0.6.18"
version = "0.6.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -103,33 +90,33 @@ dependencies = [
[[package]]
name = "anstyle"
version = "1.0.10"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
[[package]]
name = "anstyle-parse"
version = "0.2.6"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.1.2"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.8"
version = "3.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa"
checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
dependencies = [
"anstyle",
"once_cell_polyfill",
@@ -286,6 +273,26 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
[[package]]
name = "bincode"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"bincode_derive",
"serde",
"unty",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
[[package]]
name = "bitcoin"
version = "0.32.6"
@@ -381,7 +388,7 @@ dependencies = [
[[package]]
name = "brk"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"brk_cli",
"brk_computer",
@@ -393,13 +400,16 @@ dependencies = [
"brk_parser",
"brk_query",
"brk_server",
"brk_state",
"brk_store",
"brk_vec",
]
[[package]]
name = "brk_cli"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"bitcoincore-rpc",
"brk_computer",
"brk_core",
"brk_exit",
@@ -416,55 +426,55 @@ dependencies = [
"log",
"serde",
"tabled",
"tokio",
"toml",
]
[[package]]
name = "brk_computer"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"brk_core",
"brk_exit",
"brk_fetcher",
"brk_indexer",
"brk_logger",
"brk_parser",
"brk_state",
"brk_vec",
"clap",
"clap_derive",
"color-eyre",
"derive_deref",
"fjall",
"jiff",
"log",
"rayon",
"serde",
"serde_json",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "brk_core"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"bincode",
"bitcoin",
"bitcoincore-rpc",
"byteview 0.7.0",
"byteview",
"derive_deref",
"fjall",
"jiff",
"log",
"rapidhash",
"rlimit",
"serde",
"serde_bytes",
"serde_derive",
"serde_json",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "brk_exit"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"brk_logger",
"ctrlc",
@@ -473,7 +483,7 @@ dependencies = [
[[package]]
name = "brk_fetcher"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"brk_core",
"brk_logger",
@@ -486,7 +496,7 @@ dependencies = [
[[package]]
name = "brk_indexer"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
@@ -494,18 +504,17 @@ dependencies = [
"brk_exit",
"brk_logger",
"brk_parser",
"brk_store",
"brk_vec",
"byteview 0.7.0",
"color-eyre",
"fjall",
"log",
"rayon",
"zerocopy",
]
[[package]]
name = "brk_logger"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"color-eyre",
"env_logger",
@@ -515,7 +524,7 @@ dependencies = [
[[package]]
name = "brk_parser"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
@@ -530,9 +539,10 @@ dependencies = [
[[package]]
name = "brk_query"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"brk_computer",
"brk_core",
"brk_indexer",
"brk_vec",
"clap",
@@ -547,9 +557,10 @@ dependencies = [
[[package]]
name = "brk_server"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"axum",
"bitcoincore-rpc",
"brk_computer",
"brk_core",
"brk_exit",
@@ -573,9 +584,33 @@ dependencies = [
"zip",
]
[[package]]
name = "brk_state"
version = "0.0.44"
dependencies = [
"bincode",
"brk_core",
"brk_store",
"derive_deref",
"fjall",
"serde",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "brk_store"
version = "0.0.44"
dependencies = [
"arc-swap",
"brk_core",
"byteview",
"fjall",
]
[[package]]
name = "brk_vec"
version = "0.0.40"
version = "0.0.44"
dependencies = [
"arc-swap",
"brk_core",
@@ -624,9 +659,9 @@ dependencies = [
[[package]]
name = "bytecount"
version = "0.6.8"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
[[package]]
name = "byteorder"
@@ -646,12 +681,6 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]]
name = "byteview"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f996a0c883b1ae93c53ffd21857b6eaba768998dbb1f7e2f269d53dd6553256c"
[[package]]
name = "bzip2"
version = "0.5.2"
@@ -682,9 +711,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.24"
version = "1.2.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7"
checksum = "956a5e21988b87f372569b66183b78babf23ebc2e744b733e4350a752c4dafac"
dependencies = [
"jobserver",
"libc",
@@ -728,18 +757,18 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.38"
version = "4.5.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000"
checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.5.38"
version = "4.5.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120"
checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51"
dependencies = [
"anstream",
"anstyle",
@@ -767,9 +796,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "color-eyre"
version = "0.6.4"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e1761c0e16f8883bbbb8ce5990867f4f06bf11a0253da6495a04ce4b6ef0ec"
checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d"
dependencies = [
"backtrace",
"color-spantrace",
@@ -782,9 +811,9 @@ dependencies = [
[[package]]
name = "color-spantrace"
version = "0.2.2"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ddd8d5bfda1e11a501d0a7303f3bfed9aa632ebdb859be40d0fd70478ed70d5"
checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427"
dependencies = [
"once_cell",
"owo-colors",
@@ -794,9 +823,9 @@ dependencies = [
[[package]]
name = "colorchoice"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
[[package]]
name = "compact_str"
@@ -1125,12 +1154,12 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "fjall"
version = "2.10.0"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b929b3db7be7d7b4d4df67fb016fc446b8f57507b48a82e69d2a30610e460f28"
checksum = "13279146a877c2060f668bc4c477af8ef5aa42732c58dca32fcb4aff40edc5b4"
dependencies = [
"byteorder",
"byteview 0.6.1",
"byteview",
"dashmap",
"log",
"lsm-tree",
@@ -1384,12 +1413,12 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.12"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9f1e950e0d9d1d3c47184416723cf29c0d1f93bd8cccf37e4beb6b44f31710"
checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb"
dependencies = [
"bytes",
"futures-util",
"futures-core",
"http",
"http-body",
"hyper",
@@ -1591,9 +1620,9 @@ dependencies = [
[[package]]
name = "liblzma-sys"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5839bad90c3cc2e0b8c4ed8296b80e86040240f81d46b9c0e9bc8dd51ddd3af1"
checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736"
dependencies = [
"cc",
"libc",
@@ -1602,9 +1631,9 @@ dependencies = [
[[package]]
name = "libz-rs-sys"
version = "0.5.0"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a"
checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221"
dependencies = [
"zlib-rs",
]
@@ -1617,9 +1646,9 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "lock_api"
version = "0.4.12"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
dependencies = [
"autocfg",
"scopeguard",
@@ -1713,13 +1742,13 @@ dependencies = [
[[package]]
name = "mio"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
dependencies = [
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -1809,9 +1838,9 @@ checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec"
[[package]]
name = "oxc"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86a761cfb7ce8c7d07e0bb5a43d8676ae69de62b2d19a542c131784049ad22cb"
checksum = "44c130580b8327276f252c7a84b484bc35c9432e23ec41b081506d0a5bc71a28"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1827,9 +1856,9 @@ dependencies = [
[[package]]
name = "oxc-miette"
version = "2.2.1"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8c278d00ecc50ee84aba4768a7ab74eb325dff4dca8c0581495b850d53480ba"
checksum = "98b2c44324a4372caf6e3128a22744263c973e809fc598db3749ef3ff5e9fed4"
dependencies = [
"cfg-if",
"owo-colors",
@@ -1841,9 +1870,9 @@ dependencies = [
[[package]]
name = "oxc-miette-derive"
version = "2.1.2"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c0c893f53900e3fe01eca3d6d3b54085573c3e48fe25af9d57dd94ef600dcd3"
checksum = "3bd3da01a295024fa79e3b4aba14b590d91256a274ff29cc5ee8f55183b2df24"
dependencies = [
"proc-macro2",
"quote",
@@ -1852,9 +1881,9 @@ dependencies = [
[[package]]
name = "oxc_allocator"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0769ed37977fc61b76ee89fe955b184db62b07e62609388ad6e6a1eb046837e3"
checksum = "92e50218e74886659d1d13de8e6a4ff13c7e96924ed0017bc193a1feb8001b18"
dependencies = [
"allocator-api2",
"bumpalo",
@@ -1865,9 +1894,9 @@ dependencies = [
[[package]]
name = "oxc_ast"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17c6c770d1e86a9c4710d9c7974bf79e573b48efa3a764698477c11035c65738"
checksum = "3a2c546ff7887418facf35f7996f0ca2099bde75931597429b408746bbaaddcd"
dependencies = [
"bitflags",
"cow-utils",
@@ -1882,9 +1911,9 @@ dependencies = [
[[package]]
name = "oxc_ast_macros"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c08ec3f117c0370106cf39b274090ae7f0b2be48b6bcae794bdc92fd1d34f22"
checksum = "4ce8952bd09048ac55421aeecc06cc69db9f7220a25d69f3da8c6da3d95e5d6d"
dependencies = [
"phf",
"proc-macro2",
@@ -1894,9 +1923,9 @@ dependencies = [
[[package]]
name = "oxc_ast_visit"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b6fc584d5c7e5bd8394208a01d2d33025afcc8abceaed85455d642ee11db998"
checksum = "9a02f6d1e34a893acd0d6a0994ff56c861b08d472476c1bea95f9c04c6da3426"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1906,9 +1935,9 @@ dependencies = [
[[package]]
name = "oxc_cfg"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35a48bb41100a00b9a647c9aacd7fe6412d03a1c623a8285a9e514c2fa49e447"
checksum = "d869bb28669f0747c181d5f5e28d9aad063e6e4043af3222c9ab7358b9e8788f"
dependencies = [
"bitflags",
"itertools",
@@ -1921,9 +1950,9 @@ dependencies = [
[[package]]
name = "oxc_codegen"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf726a55fd107703b21b728c044a2e64c71242d35514c4e7e1c4c7dffca98220"
checksum = "88e275fbd0f26987ef15aa29fef129e75d8e3799e309797444c6ed4c5761ba2b"
dependencies = [
"bitflags",
"cow-utils",
@@ -1942,18 +1971,18 @@ dependencies = [
[[package]]
name = "oxc_data_structures"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "374638f55291a87994e37c715c9a312be3e29dfac5dd5a655847a3d4f98d2b63"
checksum = "caccf3f0c0515f32520b6207a0ef4bafd0858f94685e84a50f38c53464418e8b"
dependencies = [
"rustversion",
]
[[package]]
name = "oxc_diagnostics"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4f29efa3f609ea53f193b1ffc19f548a6ac5464f27ad20a27982d623fc5784a"
checksum = "79d4d0062c704ee11dbd56d6178f91ea17ccac05a59fe655b120a2b64d038739"
dependencies = [
"cow-utils",
"oxc-miette",
@@ -1961,9 +1990,9 @@ dependencies = [
[[package]]
name = "oxc_ecmascript"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0f469ed7c14f5a2d75f39af03517d3d0da071ae2c33c91cd75f547f434b978d"
checksum = "3e0e3cc7daeec1ed3ed75abc02a456d7046c5878c61bda972d26e39850f25298"
dependencies = [
"cow-utils",
"num-bigint",
@@ -1975,9 +2004,9 @@ dependencies = [
[[package]]
name = "oxc_estree"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bf9a77d3465740411872becbf9fc43d0462ba3adb6f2609334c1b75365dfde3"
checksum = "68bb8e3cedd84c69241a0438d14ca09294ab95049b52dc7813bba017554c96fb"
[[package]]
name = "oxc_index"
@@ -1987,9 +2016,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
[[package]]
name = "oxc_mangler"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68675862106f3ad0177dce45db81d27f406d0f27ac27541b55a60953ed5358e8"
checksum = "3038ab2f2b76130ea02ff016da89fc6c2a4a3935da50a5f1a16ebd982873dc2f"
dependencies = [
"fixedbitset",
"itertools",
@@ -2004,9 +2033,9 @@ dependencies = [
[[package]]
name = "oxc_minifier"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa89cd8c32768a12f50d949d2ad602262f3104ceca5504947850ee6c39aeaadc"
checksum = "fe473cd85aef3e71f7039c50f5fc85cc0e5a6a8fe98c49084aef4b993ffc7382"
dependencies = [
"cow-utils",
"oxc_allocator",
@@ -2026,9 +2055,9 @@ dependencies = [
[[package]]
name = "oxc_parser"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "848bc15e9fd22229224593e1468ae4d2cf8d396157b5ba9938daecbbac9a1409"
checksum = "d3bb3f8ed289796dcd7fecc4cfec895fbe29c269cf63658b63f352ce8fd7caf7"
dependencies = [
"bitflags",
"cow-utils",
@@ -2049,9 +2078,9 @@ dependencies = [
[[package]]
name = "oxc_regular_expression"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ef0bc791b7a19124561c8c285cbb0f25e7cd5e4201b9bfd770b96b5f3bcdf79"
checksum = "b8e58c2970fa7dcb10a5c757706c5824c43f13abb908a4651b221033c96c1ddc"
dependencies = [
"bitflags",
"oxc_allocator",
@@ -2065,9 +2094,9 @@ dependencies = [
[[package]]
name = "oxc_semantic"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e796ba52fa9b3c8dbc8fd3e57f5085f80dde092244098049dfa5732180304ed6"
checksum = "ee61a7e935fa88d87b02c2279ffcf0704433006fa06a635e827fcdfc53b8f741"
dependencies = [
"itertools",
"oxc_allocator",
@@ -2101,9 +2130,9 @@ dependencies = [
[[package]]
name = "oxc_span"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2c1bcdd7b1695ac01ddcde35bc7b69b72150857d14e696f82f8a38927e7ceb3"
checksum = "f290a8f173016c5b84327297bb3f2b9c158b0733703ac2f29d9e77f9b4821ddd"
dependencies = [
"compact_str",
"oxc-miette",
@@ -2114,9 +2143,9 @@ dependencies = [
[[package]]
name = "oxc_syntax"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "934fcc154f95afb172f84d250fe42833745eb06e83ced8a4e5d6669b9348528f"
checksum = "1d83f7a8a718db14fd6e8f864b70b0961e3b4ae44c9a28102262357c27aee944"
dependencies = [
"bitflags",
"cow-utils",
@@ -2135,9 +2164,9 @@ dependencies = [
[[package]]
name = "oxc_traverse"
version = "0.72.0"
version = "0.72.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e394bf07dc65a0f15eee42a5410a6e975def23c498cc7ecd38e19c286866aea"
checksum = "3a82547d2081445a2661ee858140845624f7c83b285a59f0144bb1190fcd464e"
dependencies = [
"itoa",
"oxc_allocator",
@@ -2153,30 +2182,20 @@ dependencies = [
[[package]]
name = "papergrid"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30268a8d20c2c0d126b2b6610ab405f16517f6ba9f244d8c59ac2c512a8a1ce7"
checksum = "6978128c8b51d8f4080631ceb2302ab51e32cc6e8615f735ee2f83fd269ae3f1"
dependencies = [
"ahash",
"bytecount",
"fnv",
"unicode-width",
]
[[package]]
name = "parking_lot"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.10"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
@@ -2221,9 +2240,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "petgraph"
version = "0.8.1"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a98c6720655620a521dcc722d0ad66cd8afd5d86e34a89ef691c50b7b24de06"
checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca"
dependencies = [
"fixedbitset",
"hashbrown 0.15.3",
@@ -2293,9 +2312,9 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "portable-atomic"
version = "1.11.0"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
[[package]]
name = "portable-atomic-util"
@@ -2661,9 +2680,9 @@ dependencies = [
[[package]]
name = "serde_spanned"
version = "0.6.8"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
dependencies = [
"serde",
]
@@ -2736,15 +2755,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook-registry"
version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
dependencies = [
"libc",
]
[[package]]
name = "simd-adler32"
version = "0.3.7"
@@ -2771,9 +2781,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
[[package]]
name = "socket2"
version = "0.5.9"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
@@ -2833,9 +2843,9 @@ checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
[[package]]
name = "tabled"
version = "0.19.0"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "228d124371171cd39f0f454b58f73ddebeeef3cef3207a82ffea1c29465aea43"
checksum = "e39a2ee1fbcd360805a771e1b300f78cc88fec7b8d3e2f71cd37bbf23e725c7d"
dependencies = [
"papergrid",
"tabled_derive",
@@ -2951,17 +2961,14 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.45.0"
version = "1.45.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165"
checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys 0.52.0",
@@ -2993,9 +3000,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.8.22"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae"
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
dependencies = [
"serde",
"serde_spanned",
@@ -3005,18 +3012,18 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.9"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.26"
version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.9.0",
"serde",
@@ -3028,9 +3035,9 @@ dependencies = [
[[package]]
name = "toml_write"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076"
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "tower"
@@ -3050,9 +3057,9 @@ dependencies = [
[[package]]
name = "tower-http"
version = "0.6.4"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e"
checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
dependencies = [
"async-compression",
"bitflags",
@@ -3094,9 +3101,9 @@ dependencies = [
[[package]]
name = "tracing-attributes"
version = "0.1.28"
version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"
dependencies = [
"proc-macro2",
"quote",
@@ -3105,9 +3112,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.33"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
dependencies = [
"once_cell",
"valuable",
@@ -3170,6 +3177,12 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -3189,7 +3202,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
dependencies = [
"byteorder",
"byteview 0.6.1",
"byteview",
"interval-heap",
"log",
"path-absolutize",
@@ -3211,6 +3224,12 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "vsimd"
version = "0.8.0"
@@ -3529,9 +3548,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
version = "0.5.0"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8"
checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a"
[[package]]
name = "zopfli"
+13 -6
View File
@@ -4,7 +4,8 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.0.40"
package.version = "0.0.44"
package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk"
[profile.release]
@@ -16,7 +17,9 @@ panic = "abort"
inherits = "release"
[workspace.dependencies]
arc-swap = "1.7.1"
axum = "0.8.4"
bincode = { version = "2.0.1", features = ["serde"] }
bitcoin = { version = "0.32.6", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
brk_cli = { version = "0", path = "crates/brk_cli" }
@@ -29,21 +32,25 @@ brk_logger = { version = "0", path = "crates/brk_logger" }
brk_parser = { version = "0", path = "crates/brk_parser" }
brk_query = { version = "0", path = "crates/brk_query" }
brk_server = { version = "0", path = "crates/brk_server" }
brk_state = { version = "0", path = "crates/brk_state" }
brk_store = { version = "0", path = "crates/brk_store" }
brk_vec = { version = "0", path = "crates/brk_vec" }
byteview = "0.7.0"
clap = { version = "4.5.38", features = ["string"] }
byteview = "=0.6.1"
clap = { version = "4.5.39", features = ["string"] }
clap_derive = "4.5.32"
color-eyre = "0.6.4"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall = "2.10.0"
fjall = "2.11.0"
jiff = "0.2.14"
log = { version = "0.4.27" }
minreq = { version = "2.13.4", features = ["https", "serde_json"] }
rayon = "1.10.0"
serde = { version = "1.0.219" }
serde_bytes = "0.11.17"
serde_derive = "1.0.219"
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
tabled = "0.19.0"
tabled = "0.20.0"
tokio = { version = "1.45.1", features = ["rt-multi-thread"] }
zerocopy = { version = "0.8.25" }
zerocopy-derive = "0.8.25"
+1 -1
View File
@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2025 bitcoinresearchkit, kibo.money
Copyright (c) 2025 bitcoinresearchkit, kibo.money, satonomics
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
+3 -4
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
@@ -77,6 +74,8 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
- [`brk_query`](https://crates.io/crates/brk_query): A library that finds requested datasets.
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
- [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
## Acknowledgments
@@ -92,7 +91,7 @@ Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagm
If you'd like to have your own instance hosted for you please contact [hosting@bitcoinresearchkit.org](mailto:hosting@bitcoinresearchkit.org).
- 2 separate dedicated servers (1 GB/s each) with different ISPs and Cloudflare integration for enhanced performance and optimal availability
- 99.9% SLA
- 99.99% SLA
- Configurated for speed (`raw + eager`)
- Updates delivered at your convenience
- Direct communication for feature requests and support
+7
View File
@@ -3,6 +3,7 @@ name = "brk"
description.workspace = true
license.workspace = true
readme.workspace = true
homepage.workspace = true
repository.workspace = true
edition.workspace = true
version.workspace = true
@@ -18,6 +19,8 @@ full = [
"parser",
"query",
"server",
"state",
"store",
"vec",
]
core = ["brk_core"]
@@ -29,6 +32,8 @@ logger = ["brk_logger"]
parser = ["brk_parser"]
query = ["brk_query"]
server = ["brk_server"]
state = ["brk_state"]
store = ["brk_store"]
vec = ["brk_vec"]
[dependencies]
@@ -42,6 +47,8 @@ brk_logger = { workspace = true, optional = true }
brk_parser = { workspace = true, optional = true }
brk_query = { workspace = true, optional = true }
brk_server = { workspace = true, optional = true }
brk_state = { workspace = true, optional = true }
brk_store = { workspace = true, optional = true }
brk_vec = { workspace = true, optional = true }
[package.metadata.docs.rs]
+8
View File
@@ -36,6 +36,14 @@ pub use brk_query as query;
#[doc(inline)]
pub use brk_server as server;
#[cfg(feature = "state")]
#[doc(inline)]
pub use brk_state as state;
#[cfg(feature = "store")]
#[doc(inline)]
pub use brk_store as store;
#[cfg(feature = "vec")]
#[doc(inline)]
pub use brk_vec as vec;
+4 -1
View File
@@ -4,9 +4,11 @@ description = "A command line interface to interact with the full Bitcoin Resear
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
bitcoincore-rpc = { workspace = true }
brk_computer = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
@@ -23,7 +25,8 @@ color-eyre = { workspace = true }
log = { workspace = true }
serde = { workspace = true }
tabled = { workspace = true }
toml = "0.8.22"
tokio = { workspace = true }
toml = "0.8.23"
[[bin]]
name = "brk"
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
+3 -3
View File
@@ -8,12 +8,12 @@ use crate::run::RunConfig;
pub fn query(params: QueryParams) -> color_eyre::Result<()> {
let config = RunConfig::import(None)?;
let compressed = config.compressed();
let format = config.format();
let mut indexer = Indexer::new(&config.outputsdir(), compressed, config.check_collisions())?;
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
indexer.import_vecs()?;
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed);
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
computer.import_vecs(&indexer, config.computation())?;
let query = Query::build(&indexer, &computer);
+21 -17
View File
@@ -5,14 +5,14 @@ use std::{
time::Duration,
};
use bitcoincore_rpc::{self, Auth, Client, RpcApi};
use brk_computer::Computer;
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::rpc::{self, Auth, Client, RpcApi};
use brk_server::{Server, Website, tokio};
use brk_vec::Computation;
use brk_server::{Server, Website};
use brk_vec::{Computation, Format};
use clap_derive::{Parser, ValueEnum};
use color_eyre::eyre::eyre;
use log::info;
@@ -27,9 +27,9 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
let compressed = config.compressed();
let format = config.format();
let mut indexer = Indexer::new(&config.outputsdir(), compressed, config.check_collisions())?;
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
indexer.import_stores()?;
indexer.import_vecs()?;
@@ -50,7 +50,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
};
let f = move || -> color_eyre::Result<()> {
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed);
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
computer.import_stores(&indexer)?;
computer.import_vecs(&indexer, config.computation())?;
@@ -133,18 +133,18 @@ pub struct RunConfig {
mode: Option<Mode>,
/// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
#[arg(short = 'C', long)]
#[arg(short, long)]
computation: Option<Computation>,
/// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: true, saved
#[arg(short, long, value_name = "BOOL")]
compressed: Option<bool>,
#[arg(short, long, value_name = "FORMAT")]
format: Option<Format>,
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
#[arg(short, long, value_name = "BOOL")]
#[arg(short = 'F', long, value_name = "BOOL")]
fetch: Option<bool>,
/// Website served by the server (if active), default: kibo.money, saved
/// Website served by the server (if active), default: default, saved
#[arg(short, long)]
website: Option<Website>,
@@ -204,12 +204,16 @@ impl RunConfig {
config_saved.mode = Some(mode);
}
if let Some(computation) = config_args.computation.take() {
config_saved.computation = Some(computation);
}
if let Some(fetch) = config_args.fetch.take() {
config_saved.fetch = Some(fetch);
}
if let Some(compressed) = config_args.compressed.take() {
config_saved.compressed = Some(compressed);
if let Some(format) = config_args.format.take() {
config_saved.format = Some(format);
}
if let Some(website) = config_args.website.take() {
@@ -314,7 +318,7 @@ impl RunConfig {
}
pub fn rpc(&self) -> color_eyre::Result<&'static Client> {
Ok(Box::leak(Box::new(rpc::Client::new(
Ok(Box::leak(Box::new(Client::new(
&format!(
"http://{}:{}",
self.rpcconnect().unwrap_or(&"localhost".to_string()),
@@ -414,7 +418,7 @@ impl RunConfig {
}
pub fn website(&self) -> Website {
self.website.unwrap_or(Website::KiboMoney)
self.website.unwrap_or(Website::Default)
}
pub fn fetch(&self) -> bool {
@@ -430,8 +434,8 @@ impl RunConfig {
self.computation.unwrap_or_default()
}
pub fn compressed(&self) -> bool {
self.compressed.is_none_or(|b| b)
pub fn format(&self) -> Format {
self.format.unwrap_or_default()
}
pub fn check_collisions(&self) -> bool {
+5 -7
View File
@@ -4,24 +4,22 @@ description = "A Bitcoin dataset computer, built on top of brk_indexer"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_state = { workspace = true }
brk_vec = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
+7 -7
View File
@@ -5,8 +5,8 @@ use brk_core::{default_bitcoin_path, default_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{Parser, rpc};
use brk_vec::Computation;
use brk_parser::Parser;
use brk_vec::{Computation, Format};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -15,9 +15,9 @@ pub fn main() -> color_eyre::Result<()> {
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
@@ -31,15 +31,15 @@ pub fn main() -> color_eyre::Result<()> {
let outputs_dir = _outputs_dir.as_path();
// let outputs_dir = Path::new("../../_outputs");
let compressed = false;
let format = Format::Raw;
let mut indexer = Indexer::new(outputs_dir, compressed, true)?;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
let fetcher = Fetcher::import(None)?;
let mut computer = Computer::new(outputs_dir, Some(fetcher), compressed);
let mut computer = Computer::new(outputs_dir, Some(fetcher), format);
computer.import_stores(&indexer)?;
computer.import_vecs(&indexer, Computation::Lazy)?;
+16 -16
View File
@@ -5,13 +5,12 @@
use std::path::{Path, PathBuf};
use brk_core::Version;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
pub use brk_parser::rpc;
use brk_vec::{AnyCollectableVec, Compressed, Computation};
use brk_vec::{AnyCollectableVec, Computation, Format};
mod states;
mod stores;
mod utils;
mod vecs;
@@ -26,17 +25,19 @@ pub struct Computer {
fetcher: Option<Fetcher>,
vecs: Option<Vecs>,
stores: Option<Stores>,
compressed: Compressed,
format: Format,
}
const VERSION: Version = Version::ONE;
impl Computer {
pub fn new(outputs_dir: &Path, fetcher: Option<Fetcher>, compressed: bool) -> Self {
pub fn new(outputs_dir: &Path, fetcher: Option<Fetcher>, format: Format) -> Self {
Self {
path: outputs_dir.to_owned(),
fetcher,
vecs: None,
stores: None,
compressed: Compressed::from(compressed),
format,
}
}
@@ -46,11 +47,13 @@ impl Computer {
computation: Computation,
) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(
// TODO: Give self.path, join inside import
&self.path.join("vecs/computed"),
VERSION + Version::ZERO,
indexer,
self.fetcher.is_some(),
computation,
self.compressed,
self.format,
)?);
Ok(())
}
@@ -59,7 +62,9 @@ impl Computer {
/// Clone struct instead
pub fn import_stores(&mut self, indexer: &Indexer) -> color_eyre::Result<()> {
self.stores = Some(Stores::import(
// TODO: Give self.path, join inside import
&self.path.join("stores"),
VERSION + Version::ZERO,
indexer.keyspace(),
)?);
Ok(())
@@ -74,15 +79,10 @@ impl Computer {
exit: &Exit,
) -> color_eyre::Result<()> {
info!("Computing...");
self.vecs.as_mut().unwrap().compute(
indexer,
starting_indexes,
self.fetcher.as_mut(),
exit,
)?;
Ok(())
self.vecs
.as_mut()
.unwrap()
.compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
-27
View File
@@ -1,27 +0,0 @@
use brk_core::Dollars;
use super::{RealizedState, SupplyState};
// Vecs ? probably
#[derive(Debug, Default, Clone)]
pub struct CohortState {
pub supply: SupplyState,
pub realized: Option<RealizedState>,
// pub price_to_amount: PriceToValue<Amount>, save it not rounded in fjall
}
impl CohortState {
pub fn increment(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.increment(supply_state, price.unwrap());
}
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply -= supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.decrement(supply_state, price.unwrap());
}
}
}
@@ -1,74 +0,0 @@
use super::OutputFilter;
#[derive(Default, Clone)]
pub struct OutputsByRange<T> {
pub _1d_to_1w: T,
pub _1w_to_1m: T,
pub _1m_to_3m: T,
pub _3m_to_6m: T,
pub _6m_to_1y: T,
pub _1y_to_2y: T,
pub _2y_to_3y: T,
pub _3y_to_4y: T,
pub _4y_to_5y: T,
pub _5y_to_7y: T,
pub _7y_to_10y: T,
pub _10y_to_15y: T,
}
impl<T> From<OutputsByRange<T>> for OutputsByRange<(OutputFilter, T)> {
fn from(value: OutputsByRange<T>) -> Self {
Self {
_1d_to_1w: (OutputFilter::Range(1..7), value._1d_to_1w),
_1w_to_1m: (OutputFilter::Range(7..30), value._1w_to_1m),
_1m_to_3m: (OutputFilter::Range(30..3 * 30), value._1m_to_3m),
_3m_to_6m: (OutputFilter::Range(3 * 30..6 * 30), value._3m_to_6m),
_6m_to_1y: (OutputFilter::Range(6 * 30..365), value._6m_to_1y),
_1y_to_2y: (OutputFilter::Range(365..2 * 365), value._1y_to_2y),
_2y_to_3y: (OutputFilter::Range(2 * 365..3 * 365), value._2y_to_3y),
_3y_to_4y: (OutputFilter::Range(3 * 365..4 * 365), value._3y_to_4y),
_4y_to_5y: (OutputFilter::Range(4 * 365..5 * 365), value._4y_to_5y),
_5y_to_7y: (OutputFilter::Range(5 * 365..7 * 365), value._5y_to_7y),
_7y_to_10y: (OutputFilter::Range(7 * 365..10 * 365), value._7y_to_10y),
_10y_to_15y: (OutputFilter::Range(10 * 365..15 * 365), value._10y_to_15y),
}
}
}
impl<T> OutputsByRange<T> {
pub fn as_mut_vec(&mut self) -> [&mut T; 12] {
[
&mut self._1d_to_1w,
&mut self._1w_to_1m,
&mut self._1m_to_3m,
&mut self._3m_to_6m,
&mut self._6m_to_1y,
&mut self._1y_to_2y,
&mut self._2y_to_3y,
&mut self._3y_to_4y,
&mut self._4y_to_5y,
&mut self._5y_to_7y,
&mut self._7y_to_10y,
&mut self._10y_to_15y,
]
}
}
impl<T> OutputsByRange<(OutputFilter, T)> {
pub fn vecs(&self) -> [&T; 12] {
[
&self._1d_to_1w.1,
&self._1w_to_1m.1,
&self._1m_to_3m.1,
&self._3m_to_6m.1,
&self._6m_to_1y.1,
&self._1y_to_2y.1,
&self._2y_to_3y.1,
&self._3y_to_4y.1,
&self._4y_to_5y.1,
&self._5y_to_7y.1,
&self._7y_to_10y.1,
&self._10y_to_15y.1,
]
}
}
@@ -1,14 +0,0 @@
use std::ops::Range;
use brk_core::{HalvingEpoch, OutputType};
#[derive(Debug, Clone)]
pub enum OutputFilter {
All,
To(usize),
Range(Range<usize>),
From(usize),
Size,
Epoch(HalvingEpoch),
Type(OutputType),
}
@@ -1,251 +0,0 @@
use brk_vec::StoredIndex;
use rayon::prelude::*;
use std::{collections::BTreeMap, ops::ControlFlow};
use brk_core::{Dollars, HalvingEpoch, Height, Timestamp};
mod by_epoch;
mod by_from;
mod by_range;
mod by_size;
mod by_spendable_type;
mod by_term;
mod by_type;
mod by_unspendable_type;
mod by_up_to;
// mod by_value;
mod filter;
pub use by_epoch::*;
pub use by_from::*;
pub use by_range::*;
pub use by_size::*;
pub use by_spendable_type::*;
pub use by_term::*;
pub use by_type::*;
pub use by_unspendable_type::*;
pub use by_up_to::*;
// pub use by_value::*;
pub use filter::*;
use crate::vecs;
use super::{BlockState, Transacted};
#[derive(Default, Clone)]
pub struct Outputs<T> {
pub all: T,
pub by_term: OutputsByTerm<T>,
pub by_up_to: OutputsByUpTo<T>,
pub by_from: OutputsByFrom<T>,
pub by_range: OutputsByRange<T>,
pub by_epoch: OutputsByEpoch<T>,
pub by_type: OutputsBySpendableType<T>,
pub by_size: OutputsBySize<T>,
// // Needs whole UTXO set, TODO later
// // pub by_value: OutputsByValue<T>,
}
impl<T> Outputs<T> {
pub fn as_mut_vec(&mut self) -> Vec<&mut T> {
[&mut self.all]
.into_iter()
.chain(self.by_term.as_mut_vec())
.chain(self.by_up_to.as_mut_vec())
.chain(self.by_from.as_mut_vec())
.chain(self.by_range.as_mut_vec())
.chain(self.by_epoch.as_mut_vec())
.chain(self.by_size.as_mut_vec())
.chain(self.by_type.as_mut_vec())
// // .chain(self.by_value.as_mut_vec())
.collect::<Vec<_>>()
}
}
impl Outputs<(OutputFilter, vecs::utxos::cohort::Vecs)> {
pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
if chain_state.is_empty() {
return;
}
let prev_timestamp = chain_state.last().unwrap().timestamp;
self.by_term
.as_mut_vec()
.into_par_iter()
.chain(self.by_up_to.as_mut_vec())
.chain(self.by_from.as_mut_vec())
.chain(self.by_range.as_mut_vec())
.for_each(|(filter, v)| {
let state = &mut v.state;
let mut check_days_old = |days_old: usize| -> bool {
match filter {
OutputFilter::From(from) => *from <= days_old,
OutputFilter::To(to) => *to > days_old,
OutputFilter::Range(range) => range.contains(&days_old),
OutputFilter::All
| OutputFilter::Epoch(_)
| OutputFilter::Size
| OutputFilter::Type(_) => unreachable!(),
}
};
let _ = chain_state
.iter()
.try_for_each(|block_state| -> ControlFlow<()> {
let prev_days_old = block_state
.timestamp
.difference_in_days_between(prev_timestamp);
let days_old = block_state.timestamp.difference_in_days_between(timestamp);
if prev_days_old == days_old {
return ControlFlow::Continue(());
}
let is = check_days_old(days_old);
let was = check_days_old(prev_days_old);
if is && !was {
state.increment(&block_state.supply, block_state.price);
} else if was && !is {
state.decrement(&block_state.supply, block_state.price);
}
ControlFlow::Continue(())
});
});
}
pub fn send(
&mut self,
height_to_sent: BTreeMap<Height, Transacted>,
chain_state: &[BlockState],
) {
let mut time_based_vecs = self
.by_term
.as_mut_vec()
.into_iter()
.chain(self.by_up_to.as_mut_vec())
.chain(self.by_from.as_mut_vec())
.chain(self.by_range.as_mut_vec())
.chain(self.by_epoch.as_mut_vec())
.collect::<Vec<_>>();
let last_timestamp = chain_state.last().unwrap().timestamp;
height_to_sent.into_iter().for_each(|(height, sent)| {
let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
let price = block_state.price;
let days_old = block_state
.timestamp
.difference_in_days_between(last_timestamp);
self.all.1.state.decrement(&sent.spendable_supply, price);
time_based_vecs
.iter_mut()
.filter(|(filter, _)| match filter {
OutputFilter::From(from) => *from <= days_old,
OutputFilter::To(to) => *to > days_old,
OutputFilter::Range(range) => range.contains(&days_old),
OutputFilter::Epoch(epoch) => *epoch == HalvingEpoch::from(height),
_ => unreachable!(),
})
.for_each(|(_, vecs)| {
vecs.state.decrement(&sent.spendable_supply, price);
});
sent.by_type.spendable.as_typed_vec().into_iter().for_each(
|(output_type, supply_state)| {
self.by_type
.get_mut(output_type)
.1
.state
.decrement(supply_state, price)
},
);
sent.by_size.into_iter().for_each(|(group, supply_state)| {
self.by_size
.get_mut(group)
.1
.state
.decrement(&supply_state, price);
});
});
}
pub fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
let supply_state = received.spendable_supply;
[
&mut self.all.1,
&mut self.by_term.short.1,
&mut self.by_epoch.mut_vec_from_height(height).1,
// Skip from and range as can't receive in the past
]
.into_iter()
.chain(self.by_up_to.as_mut_vec().map(|(_, v)| v))
.for_each(|v| {
v.state.increment(&supply_state, price);
});
self.by_type
.as_mut_vec()
.into_iter()
.for_each(|(filter, vecs)| {
let output_type = match filter {
OutputFilter::Type(output_type) => *output_type,
_ => unreachable!(),
};
vecs.state
.increment(received.by_type.get(output_type), price)
});
received
.by_size
.into_iter()
.for_each(|(group, supply_state)| {
self.by_size
.get_mut(group)
.1
.state
.increment(&supply_state, price);
});
}
}
impl<T> Outputs<(OutputFilter, T)> {
pub fn vecs(&self) -> Vec<&T> {
[&self.all.1]
.into_iter()
.chain(self.by_term.vecs())
.chain(self.by_up_to.vecs())
.chain(self.by_from.vecs())
.chain(self.by_range.vecs())
.chain(self.by_epoch.vecs())
.chain(self.by_size.vecs())
// // .chain(self.by_value.vecs())
.chain(self.by_type.vecs())
.collect::<Vec<_>>()
}
}
impl<T> From<Outputs<T>> for Outputs<(OutputFilter, T)> {
fn from(value: Outputs<T>) -> Self {
Self {
all: (OutputFilter::All, value.all),
by_term: OutputsByTerm::from(value.by_term),
by_up_to: OutputsByUpTo::from(value.by_up_to),
by_from: OutputsByFrom::from(value.by_from),
by_range: OutputsByRange::from(value.by_range),
by_epoch: OutputsByEpoch::from(value.by_epoch),
by_size: OutputsBySize::from(value.by_size),
// // Needs whole UTXO set, TODO later
// // by_value: OutputsByValue<T>,
by_type: OutputsBySpendableType::from(value.by_type),
}
}
}
@@ -1,49 +0,0 @@
use brk_core::{Bitcoin, CheckedSub, Dollars};
use super::SupplyState;
#[derive(Debug, Default, Clone)]
pub struct RealizedState {
pub realized_cap: Dollars,
// pub realized_profit: Dollars,
// pub realized_loss: Dollars,
// pub value_created: Dollars,
// pub adjusted_value_created: Dollars,
// pub value_destroyed: Dollars,
// pub adjusted_value_destroyed: Dollars,
}
impl RealizedState {
pub const NAN: Self = Self {
realized_cap: Dollars::NAN,
// realized_profit: Dollars::NAN,
// realized_loss: Dollars::NAN,
// value_created: Dollars::NAN,
// adjusted_value_created: Dollars::NAN,
// value_destroyed: Dollars::NAN,
// adjusted_value_destroyed: Dollars::NAN,
};
pub fn increment(&mut self, supply_state: &SupplyState, price: Dollars) {
if supply_state.value.is_not_zero() {
if self.realized_cap == Dollars::NAN {
self.realized_cap = Dollars::ZERO;
// self.realized_profit = Dollars::ZERO;
// self.realized_loss = Dollars::ZERO;
// self.value_created = Dollars::ZERO;
// self.adjusted_value_created = Dollars::ZERO;
// self.value_destroyed = Dollars::ZERO;
// self.adjusted_value_destroyed = Dollars::ZERO;
}
self.realized_cap += price * Bitcoin::from(supply_state.value);
}
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) {
self.realized_cap = self
.realized_cap
.checked_sub(price * Bitcoin::from(supply_state.value))
.unwrap();
}
}
+6 -3
View File
@@ -1,7 +1,10 @@
use std::path::Path;
use brk_core::Version;
use fjall::TransactionalKeyspace;
const _VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Stores {
// pub address_to_utxos_received: Store<AddressIndexOutputIndex, Unit>,
@@ -9,18 +12,18 @@ pub struct Stores {
}
impl Stores {
pub fn import(_: &Path, _: &TransactionalKeyspace) -> color_eyre::Result<Self> {
pub fn import(_: &Path, _: Version, _: &TransactionalKeyspace) -> color_eyre::Result<Self> {
// let address_to_utxos_received = Store::import(
// keyspace.clone(),
// path,
// "address_to_utxos_received",
// Version::ZERO,
// version + VERSION + Version::ZERO,
// )?;
// let address_to_utxos_spent = Store::import(
// keyspace.clone(),
// path,
// "address_to_utxos_spent",
// Version::ZERO,
// version + VERSION + Version::ZERO,
// )?;
Ok(Self {
+44 -28
View File
@@ -2,12 +2,11 @@ use std::{fs, path::Path};
use brk_core::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
Timestamp, Weight,
Timestamp, Version, Weight,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
use super::{
Indexes,
@@ -15,6 +14,8 @@ use super::{
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub height_to_interval: EagerVec<Height, Timestamp>,
@@ -32,8 +33,9 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
version: Version,
_computation: Computation,
compressed: Compressed,
format: Format,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -41,22 +43,23 @@ impl Vecs {
height_to_interval: EagerVec::forced_import(
path,
"interval",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
path,
"timestamp",
Version::ZERO,
compressed,
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_first(),
)?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
path,
"block_interval",
false,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_minmax()
@@ -66,46 +69,59 @@ impl Vecs {
path,
"block_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
path,
"block_weight",
false,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
path,
"block_size",
false,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
path,
"vbytes",
version + VERSION + Version::ZERO,
format,
)?,
height_to_vbytes: EagerVec::forced_import(path, "vbytes", Version::ZERO, compressed)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
path,
"block_vbytes",
false,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
path,
"timestamp",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_timestamp: EagerVec::forced_import(
path,
"timestamp",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
})
}
@@ -117,7 +133,7 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.timeindexes_to_timestamp.compute(
self.timeindexes_to_timestamp.compute_all(
indexer,
indexes,
starting_indexes,
+14 -11
View File
@@ -1,9 +1,9 @@
use std::{fs, path::Path};
use brk_core::StoredU8;
use brk_core::{StoredU8, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyVec, Compressed, Computation, Version};
use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
use super::{
Indexes,
@@ -11,6 +11,8 @@ use super::{
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub _0: ComputedVecsFromHeight<StoredU8>,
@@ -22,8 +24,9 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
version: Version,
_computation: Computation,
compressed: Compressed,
format: Format,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -32,32 +35,32 @@ impl Vecs {
path,
"0",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
_1: ComputedVecsFromHeight::forced_import(
path,
"1",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
_50: ComputedVecsFromHeight::forced_import(
path,
"50",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
_100: ComputedVecsFromHeight::forced_import(
path,
"100",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
})
+160 -98
View File
@@ -2,12 +2,12 @@ use std::{fs, path::Path};
use brk_core::{
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, WeekIndex, YearIndex,
OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
use super::{
Indexes,
@@ -66,13 +66,14 @@ pub struct Vecs {
}
const VERSION: Version = Version::ZERO;
const VERSION_IN_SATS: Version = Version::ONE;
const VERSION_IN_SATS: Version = Version::ZERO;
impl Vecs {
pub fn forced_import(
path: &Path,
version: Version,
_computation: Computation,
compressed: Compressed,
format: Format,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -84,237 +85,280 @@ impl Vecs {
dateindex_to_ohlc_in_cents: EagerVec::forced_import(
&fetched_path,
"ohlc_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?,
dateindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
dateindex_to_close_in_cents: EagerVec::forced_import(
path,
"close_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_high_in_cents: EagerVec::forced_import(
path,
"high_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_low_in_cents: EagerVec::forced_import(
path,
"low_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_open_in_cents: EagerVec::forced_import(
path,
"open_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc_in_cents: EagerVec::forced_import(
&fetched_path,
"ohlc_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?,
height_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
height_to_close_in_cents: EagerVec::forced_import(
path,
"close_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_high_in_cents: EagerVec::forced_import(
path,
"high_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_low_in_cents: EagerVec::forced_import(
path,
"low_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_open_in_cents: EagerVec::forced_import(
path,
"open_in_cents",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
path,
"open",
Version::ZERO,
compressed,
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_first(),
)?,
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
path,
"high",
Version::ZERO,
compressed,
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_max(),
)?,
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
path,
"low",
Version::ZERO,
compressed,
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_min(),
)?,
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
path,
"close",
Version::ZERO,
compressed,
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"open_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_first(),
)?,
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"high_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_max(),
)?,
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"low_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_min(),
)?,
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"close_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
path,
"open",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_first(),
)?,
chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
path,
"high",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_max(),
)?,
chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
path,
"low",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_min(),
)?,
chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
path,
"close",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
"open_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_first(),
)?,
chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
"high_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_max(),
)?,
chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
"low_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_min(),
)?,
chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
"close_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
weekindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?,
weekindex_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
difficultyepoch_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
monthindex_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?,
monthindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
quarterindex_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?,
quarterindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
yearindex_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?,
yearindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
// halvingepoch_to_ohlc: StorableVec::forced_import(path,
// "halvingepoch_to_ohlc"), Version::ZERO, compressed)?,
decadeindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?,
// "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
decadeindex_to_ohlc: EagerVec::forced_import(
path,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
"ohlc_in_sats",
VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
})
}
@@ -427,7 +471,7 @@ impl Vecs {
exit,
)?;
self.timeindexes_to_close.compute(
self.timeindexes_to_close.compute_all(
indexer,
indexes,
starting_indexes,
@@ -442,7 +486,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_high.compute(
self.timeindexes_to_high.compute_all(
indexer,
indexes,
starting_indexes,
@@ -457,7 +501,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_low.compute(
self.timeindexes_to_low.compute_all(
indexer,
indexes,
starting_indexes,
@@ -472,7 +516,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_open.compute(
self.timeindexes_to_open.compute_all(
indexer,
indexes,
starting_indexes,
@@ -740,7 +784,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_open_in_sats.compute(
self.timeindexes_to_open_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -748,14 +792,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_open.dateindex,
self.timeindexes_to_open.dateindex.as_ref().unwrap(),
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
exit,
)
},
)?;
self.timeindexes_to_high_in_sats.compute(
self.timeindexes_to_high_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -763,14 +807,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_low.dateindex,
self.timeindexes_to_low.dateindex.as_ref().unwrap(),
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
exit,
)
},
)?;
self.timeindexes_to_low_in_sats.compute(
self.timeindexes_to_low_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -778,14 +822,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_high.dateindex,
self.timeindexes_to_high.dateindex.as_ref().unwrap(),
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
exit,
)
},
)?;
self.timeindexes_to_close_in_sats.compute(
self.timeindexes_to_close_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -793,7 +837,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_close.dateindex,
self.timeindexes_to_close.dateindex.as_ref().unwrap(),
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
exit,
)
@@ -820,12 +864,30 @@ impl Vecs {
exit,
)?;
let mut dateindex_first_iter = self.timeindexes_to_open_in_sats.dateindex.iter();
let mut dateindex_max_iter = self.timeindexes_to_high_in_sats.dateindex.iter();
let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter();
let mut dateindex_first_iter = self
.timeindexes_to_open_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
let mut dateindex_max_iter = self
.timeindexes_to_high_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
let mut dateindex_min_iter = self
.timeindexes_to_low_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
self.dateindex_to_ohlc_in_sats.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_close_in_sats.dateindex,
self.timeindexes_to_close_in_sats
.dateindex
.as_ref()
.unwrap(),
|(i, close, ..)| {
(
i,
+80 -82
View File
@@ -1,11 +1,8 @@
use std::path::Path;
use brk_core::{CheckedSub, StoredUsize};
use brk_core::{CheckedSub, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, StoredIndex, StoredType,
Version,
};
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::utils::get_percentile;
@@ -29,7 +26,7 @@ where
pub _10p: Option<Box<EagerVec<I, T>>>,
pub min: Option<Box<EagerVec<I, T>>>,
pub last: Option<Box<EagerVec<I, T>>>,
pub total: Option<Box<EagerVec<I, T>>>,
pub cumulative: Option<Box<EagerVec<I, T>>>,
}
const VERSION: Version = Version::ZERO;
@@ -43,7 +40,7 @@ where
path: &Path,
name: &str,
version: Version,
compressed: Compressed,
format: Format,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let only_one_active = options.is_only_one_active();
@@ -68,24 +65,21 @@ where
}
};
let version = VERSION + version;
let s = Self {
first: options.first.then(|| {
Box::new(
EagerVec::forced_import(
path,
&maybe_prefix("first"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
}),
last: options.last.then(|| {
Box::new(
EagerVec::forced_import(path, name, version + Version::ZERO, compressed)
.unwrap(),
EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(),
)
}),
min: options.min.then(|| {
@@ -93,8 +87,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("min"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -104,8 +98,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("max"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -115,8 +109,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("median"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -126,8 +120,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("average"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -137,19 +131,19 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("sum"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
}),
total: options.total.then(|| {
cumulative: options.cumulative.then(|| {
Box::new(
EagerVec::forced_import(
path,
&prefix("total"),
version + Version::ZERO,
compressed,
&prefix("cumulative"),
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -159,8 +153,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("90p"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -170,8 +164,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("75p"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -181,8 +175,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("25p"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -192,8 +186,8 @@ where
EagerVec::forced_import(
path,
&maybe_suffix("10p"),
version + Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)
.unwrap(),
)
@@ -209,20 +203,22 @@ where
source: &impl AnyIterableVec<I, T>,
exit: &Exit,
) -> Result<()> {
if self.total.is_none() {
if self.cumulative.is_none() {
return Ok(());
};
self.validate_computed_version_or_reset_file(source.version())?;
let index = self.starting_index(max_from);
let total_vec = self.total.as_mut().unwrap();
let cumulative_vec = self.cumulative.as_mut().unwrap();
let mut total = index.decremented().map_or(T::from(0_usize), |index| {
total_vec.iter().unwrap_get_inner(index)
let mut cumulative = index.decremented().map_or(T::from(0_usize), |index| {
cumulative_vec.iter().unwrap_get_inner(index)
});
source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
total = total.clone() + v.into_inner();
total_vec.forced_push_at(i, total.clone(), exit)
cumulative = cumulative.clone() + v.into_inner();
cumulative_vec.forced_push_at(i, cumulative.clone(), exit)
})?;
self.safe_flush(exit)?;
@@ -250,11 +246,11 @@ where
let mut count_indexes_iter = count_indexes.iter();
let mut source_iter = source.iter();
let total_vec = self.total.as_mut();
let cumulative_vec = self.cumulative.as_mut();
let mut total = total_vec.map(|total_vec| {
let mut cumulative = cumulative_vec.map(|cumulative_vec| {
index.decremented().map_or(T::from(0_usize), |index| {
total_vec.iter().unwrap_get_inner(index)
cumulative_vec.iter().unwrap_get_inner(index)
})
});
@@ -288,8 +284,9 @@ where
last.forced_push_at(index, v, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
let needs_average_sum_or_cumulative =
needs_sum_or_cumulative || self.average.is_some();
let needs_sorted = self.max.is_some()
|| self._90p.is_some()
|| self._75p.is_some()
@@ -297,7 +294,7 @@ where
|| self._25p.is_some()
|| self._10p.is_some()
|| self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
if needs_values {
source_iter.set(first_index);
@@ -358,7 +355,7 @@ where
}
}
if needs_average_sum_or_total {
if needs_average_sum_or_cumulative {
let len = values.len();
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
@@ -367,15 +364,15 @@ where
average.forced_push_at(i, avg, exit)?;
}
if needs_sum_or_total {
if needs_sum_or_cumulative {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
}
if let Some(total_vec) = self.total.as_mut() {
let t = total.as_ref().unwrap().clone() + sum;
total.replace(t.clone());
total_vec.forced_push_at(i, t, exit)?;
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
cumulative_vec.forced_push_at(i, t, exit)?;
}
}
}
@@ -425,9 +422,9 @@ where
let mut source_average_iter = source.average.as_ref().map(|f| f.iter());
let mut source_sum_iter = source.sum.as_ref().map(|f| f.iter());
let mut total = self.total.as_mut().map(|total_vec| {
let mut cumulative = self.cumulative.as_mut().map(|cumulative_vec| {
index.decremented().map_or(T::from(0_usize), |index| {
total_vec.iter().unwrap_get_inner(index)
cumulative_vec.iter().unwrap_get_inner(index)
})
});
@@ -459,10 +456,11 @@ where
last.forced_push_at(index, v, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
let needs_average_sum_or_cumulative =
needs_sum_or_cumulative || self.average.is_some();
let needs_sorted = self.max.is_some() || self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
if needs_values {
if needs_sorted {
@@ -489,7 +487,7 @@ where
}
}
if needs_average_sum_or_total {
if needs_average_sum_or_cumulative {
if let Some(average) = self.average.as_mut() {
let source_average_iter = source_average_iter.as_mut().unwrap();
source_average_iter.set(first_index);
@@ -499,14 +497,14 @@ where
.collect::<Vec<_>>();
let len = values.len();
let total = values.into_iter().fold(T::from(0), |a, b| a + b);
// TODO: Multiply by count then divide by total
let cumulative = values.into_iter().fold(T::from(0), |a, b| a + b);
// TODO: Multiply by count then divide by cumulative
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
let avg = total / len;
let avg = cumulative / len;
average.forced_push_at(i, avg, exit)?;
}
if needs_sum_or_total {
if needs_sum_or_cumulative {
let source_sum_iter = source_sum_iter.as_mut().unwrap();
source_sum_iter.set(first_index);
let values = source_sum_iter
@@ -520,10 +518,10 @@ where
sum_vec.forced_push_at(i, sum.clone(), exit)?;
}
if let Some(total_vec) = self.total.as_mut() {
let t = total.as_ref().unwrap().clone() + sum;
total.replace(t.clone());
total_vec.forced_push_at(i, t, exit)?;
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
cumulative_vec.forced_push_at(i, t, exit)?;
}
}
}
@@ -583,8 +581,8 @@ where
self.last.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_total(&self) -> &EagerVec<I, T> {
self.total.as_ref().unwrap()
pub fn unwrap_cumulative(&self) -> &EagerVec<I, T> {
self.cumulative.as_ref().unwrap()
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -611,8 +609,8 @@ where
if let Some(sum) = self.sum.as_ref() {
v.push(sum.as_ref());
}
if let Some(total) = self.total.as_ref() {
v.push(total.as_ref());
if let Some(cumulative) = self.cumulative.as_ref() {
v.push(cumulative.as_ref());
}
if let Some(_90p) = self._90p.as_ref() {
v.push(_90p.as_ref());
@@ -652,8 +650,8 @@ where
if let Some(sum) = self.sum.as_mut() {
sum.safe_flush(exit)?;
}
if let Some(total) = self.total.as_mut() {
total.safe_flush(exit)?;
if let Some(cumulative) = self.cumulative.as_mut() {
cumulative.safe_flush(exit)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.safe_flush(exit)?;
@@ -693,8 +691,8 @@ where
if let Some(sum) = self.sum.as_mut() {
sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
}
if let Some(total) = self.total.as_mut() {
total.validate_computed_version_or_reset_file(Version::ZERO + version)?;
if let Some(cumulative) = self.cumulative.as_mut() {
cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
@@ -726,7 +724,7 @@ pub struct StorableVecGeneatorOptions {
min: bool,
first: bool,
last: bool,
total: bool,
cumulative: bool,
}
impl StorableVecGeneatorOptions {
@@ -790,8 +788,8 @@ impl StorableVecGeneatorOptions {
self
}
pub fn add_total(mut self) -> Self {
self.total = true;
pub fn add_cumulative(mut self) -> Self {
self.cumulative = true;
self
}
@@ -850,8 +848,8 @@ impl StorableVecGeneatorOptions {
}
#[allow(unused)]
pub fn rm_total(mut self) -> Self {
self.total = false;
pub fn rm_cumulative(mut self) -> Self {
self.cumulative = false;
self
}
@@ -892,7 +890,7 @@ impl StorableVecGeneatorOptions {
self.min,
self.first,
self.last,
self.total,
self.cumulative,
]
.iter()
.filter(|b| **b)
@@ -902,7 +900,7 @@ impl StorableVecGeneatorOptions {
pub fn copy_self_extra(&self) -> Self {
Self {
total: self.total,
cumulative: self.cumulative,
..Self::default()
}
}
@@ -1,9 +1,11 @@
use std::path::Path;
use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex};
use brk_core::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
use crate::vecs::{Indexes, indexes};
@@ -14,7 +16,7 @@ pub struct ComputedVecsFromDateIndex<T>
where
T: ComputedType + PartialOrd,
{
pub dateindex: EagerVec<DateIndex, T>,
pub dateindex: Option<EagerVec<DateIndex, T>>,
pub dateindex_extra: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
pub monthindex: ComputedVecBuilder<MonthIndex, T>,
@@ -32,40 +34,67 @@ where
pub fn forced_import(
path: &Path,
name: &str,
compute_source: bool,
version: Version,
compressed: Compressed,
format: Format,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let version = VERSION + version;
let dateindex = compute_source.then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
});
let dateindex_extra = ComputedVecBuilder::forced_import(
path,
name,
version,
compressed,
version + VERSION + Version::ZERO,
format,
options.copy_self_extra(),
)?;
let options = options.remove_percentiles();
Ok(Self {
dateindex: EagerVec::forced_import(path, name, version, compressed)?,
dateindex,
dateindex_extra,
weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
weekindex: ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
monthindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
quarterindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
yearindex: ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
decadeindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
})
}
pub fn compute<F>(
pub fn compute_all<F>(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
@@ -83,14 +112,15 @@ where
) -> Result<()>,
{
compute(
&mut self.dateindex,
self.dateindex.as_mut().unwrap(),
indexer,
indexes,
starting_indexes,
exit,
)?;
self.compute_rest(indexes, starting_indexes, exit)
let dateindex: Option<&EagerVec<DateIndex, T>> = None;
self.compute_rest(indexes, starting_indexes, exit, dateindex)
}
pub fn compute_rest(
@@ -98,25 +128,49 @@ where
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
) -> color_eyre::Result<()> {
self.dateindex_extra
.extend(starting_indexes.dateindex, &self.dateindex, exit)?;
if let Some(dateindex) = dateindex {
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
&self.dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.weekindex.compute(
starting_indexes.weekindex,
dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
&self.dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
} else {
let dateindex = self.dateindex.as_ref().unwrap();
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
}
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
@@ -147,7 +201,9 @@ where
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
vec![&self.dateindex as &dyn AnyCollectableVec],
self.dateindex
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.dateindex_extra.vecs(),
self.weekindex.vecs(),
self.monthindex.vecs(),
@@ -1,11 +1,12 @@
use std::path::Path;
use brk_core::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex,
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, Version,
WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, Version};
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
use crate::vecs::{Indexes, indexes};
@@ -16,7 +17,7 @@ pub struct ComputedVecsFromHeight<T>
where
T: ComputedType + PartialOrd,
{
pub height: Option<Box<EagerVec<Height, T>>>,
pub height: Option<EagerVec<Height, T>>,
pub height_extra: ComputedVecBuilder<Height, T>,
pub dateindex: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
@@ -40,24 +41,28 @@ where
name: &str,
compute_source: bool,
version: Version,
compressed: Compressed,
format: Format,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let version = VERSION + version;
let height = compute_source
.then(|| Box::new(EagerVec::forced_import(path, name, version, compressed).unwrap()));
let height = compute_source.then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
});
let height_extra = ComputedVecBuilder::forced_import(
path,
name,
version,
compressed,
version + VERSION + Version::ZERO,
format,
options.copy_self_extra(),
)?;
let dateindex =
ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
let dateindex = ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?;
let options = options.remove_percentiles();
@@ -65,20 +70,48 @@ where
height,
height_extra,
dateindex,
weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
weekindex: ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
difficultyepoch: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
monthindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
quarterindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
yearindex: ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
decadeindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
})
}
@@ -133,7 +166,7 @@ where
exit,
)?;
} else {
let height = self.height.as_ref().unwrap().as_ref();
let height = self.height.as_ref().unwrap();
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -202,7 +235,7 @@ where
[
self.height
.as_ref()
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.height_extra.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),
@@ -1,9 +1,9 @@
use std::path::Path;
use brk_core::{DifficultyEpoch, Height};
use brk_core::{DifficultyEpoch, Height, Result, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
use brk_vec::{AnyCollectableVec, EagerVec, Format};
use crate::vecs::{Indexes, indexes};
@@ -31,18 +31,17 @@ where
path: &Path,
name: &str,
version: Version,
compressed: Compressed,
format: Format,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let version = VERSION + version;
let height = EagerVec::forced_import(path, name, version, compressed)?;
let height =
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?;
let height_extra = ComputedVecBuilder::forced_import(
path,
name,
version,
compressed,
version + VERSION + Version::ZERO,
format,
options.copy_self_extra(),
)?;
@@ -52,9 +51,13 @@ where
height,
height_extra,
difficultyepoch: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
})
}
@@ -2,13 +2,12 @@ use std::path::Path;
use brk_core::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, TxIndex, WeekIndex, YearIndex,
Result, Sats, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, CollectableVec, Compressed, EagerVec, Result, StoredIndex,
VecIterator, Version,
AnyCollectableVec, AnyVec, CollectableVec, EagerVec, Format, StoredIndex, VecIterator,
};
use crate::vecs::{Indexes, fetched, indexes};
@@ -44,36 +43,78 @@ where
name: &str,
compute_source: bool,
version: Version,
compressed: Compressed,
format: Format,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let version = VERSION + version;
let txindex = compute_source.then(|| {
Box::new(
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
.unwrap(),
)
});
let txindex = compute_source
.then(|| Box::new(EagerVec::forced_import(path, name, version, compressed).unwrap()));
let height = ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
let height = ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?;
let options = options.remove_percentiles();
Ok(Self {
txindex,
height,
dateindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
dateindex: ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
weekindex: ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
difficultyepoch: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
monthindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
quarterindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
yearindex: ComputedVecBuilder::forced_import(
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
decadeindex: ComputedVecBuilder::forced_import(
path, name, version, compressed, options,
path,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
})
}
@@ -383,12 +424,12 @@ impl ComputedVecsFromTxindex<Bitcoin> {
exit,
)?;
}
if let Some(total) = self.height.total.as_mut() {
total.forced_push_at(
if let Some(cumulative) = self.height.cumulative.as_mut() {
cumulative.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_total()
.unwrap_cumulative()
.into_iter()
.unwrap_get_inner(height),
),
@@ -566,13 +607,13 @@ impl ComputedVecsFromTxindex<Dollars> {
exit,
)?;
}
if let Some(total) = self.height.total.as_mut() {
total.forced_push_at(
if let Some(cumulative) = self.height.cumulative.as_mut() {
cumulative.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_total()
.unwrap_cumulative()
.into_iter()
.unwrap_get_inner(height),
exit,
@@ -5,8 +5,10 @@ mod from_height_strict;
mod from_txindex;
mod ratio_from_dateindex;
mod r#type;
mod value_from_dateindex;
mod value_from_height;
mod value_from_txindex;
mod value_height;
pub use builder::*;
pub use from_dateindex::*;
@@ -15,5 +17,7 @@ pub use from_height_strict::*;
pub use from_txindex::*;
pub use ratio_from_dateindex::*;
use r#type::*;
pub use value_from_dateindex::*;
pub use value_from_height::*;
pub use value_from_txindex::*;
pub use value_height::*;
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,176 @@
use std::path::Path;
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
use super::StorableVecGeneatorOptions;
#[derive(Clone)]
pub struct ComputedValueVecsFromDateIndex {
pub sats: ComputedVecsFromDateIndex<Sats>,
pub bitcoin: ComputedVecsFromDateIndex<Bitcoin>,
pub dollars: Option<ComputedVecsFromDateIndex<Dollars>>,
}
const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromDateIndex {
pub fn forced_import(
path: &Path,
name: &str,
compute_source: bool,
version: Version,
format: Format,
options: StorableVecGeneatorOptions,
compute_dollars: bool,
) -> color_eyre::Result<Self> {
Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import(
path,
name,
compute_source,
version + VERSION,
format,
options,
)?,
bitcoin: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_in_btc"),
true,
version + VERSION,
format,
options,
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_in_usd"),
true,
version + VERSION,
format,
options,
)
.unwrap()
}),
})
}
pub fn compute_all<F>(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
where
F: FnMut(
&mut EagerVec<DateIndex, Sats>,
&Indexer,
&indexes::Vecs,
&Indexes,
&Exit,
) -> Result<()>,
{
compute(
self.sats.dateindex.as_mut().unwrap(),
indexer,
indexes,
starting_indexes,
exit,
)?;
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, dateindex)?;
Ok(())
}
pub fn compute_rest(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl CollectableVec<DateIndex, Sats>>,
) -> color_eyre::Result<()> {
if let Some(dateindex) = dateindex {
self.sats
.compute_rest(indexes, starting_indexes, exit, Some(dateindex))?;
self.bitcoin.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_from_sats(starting_indexes.dateindex, dateindex, exit)
},
)?;
} else {
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
self.sats
.compute_rest(indexes, starting_indexes, exit, dateindex)?;
self.bitcoin.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_from_sats(
starting_indexes.dateindex,
self.sats.dateindex.as_ref().unwrap(),
exit,
)
},
)?;
}
let dateindex_to_bitcoin = self.bitcoin.dateindex.as_ref().unwrap();
let dateindex_to_close = fetched
.as_ref()
.unwrap()
.timeindexes_to_close
.dateindex
.as_ref()
.unwrap();
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_from_bitcoin(
starting_indexes.dateindex,
dateindex_to_bitcoin,
dateindex_to_close,
exit,
)
},
)?;
}
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.sats.vecs(),
self.bitcoin.vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
@@ -1,11 +1,9 @@
use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Sats};
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version,
};
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
use crate::vecs::{Indexes, fetched, indexes};
@@ -18,7 +16,7 @@ pub struct ComputedValueVecsFromHeight {
pub dollars: Option<ComputedVecsFromHeight<Dollars>>,
}
const VERSION: Version = Version::ONE;
const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromHeight {
pub fn forced_import(
@@ -26,7 +24,7 @@ impl ComputedValueVecsFromHeight {
name: &str,
compute_source: bool,
version: Version,
compressed: Compressed,
format: Format,
options: StorableVecGeneatorOptions,
compute_dollars: bool,
) -> color_eyre::Result<Self> {
@@ -35,16 +33,16 @@ impl ComputedValueVecsFromHeight {
path,
name,
compute_source,
VERSION + version,
compressed,
version + VERSION,
format,
options,
)?,
bitcoin: ComputedVecsFromHeight::forced_import(
path,
&format!("{name}_in_btc"),
true,
VERSION + version,
compressed,
version + VERSION,
format,
options,
)?,
dollars: compute_dollars.then(|| {
@@ -52,8 +50,8 @@ impl ComputedValueVecsFromHeight {
path,
&format!("{name}_in_usd"),
true,
VERSION + version,
compressed,
version + VERSION,
format,
options,
)
.unwrap()
@@ -129,14 +127,14 @@ impl ComputedValueVecsFromHeight {
|v, _, _, starting_indexes, exit| {
v.compute_from_sats(
starting_indexes.height,
self.sats.height.as_ref().unwrap().as_ref(),
self.sats.height.as_ref().unwrap(),
exit,
)
},
)?;
}
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap().as_ref();
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
@@ -1,11 +1,11 @@
use std::path::Path;
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex};
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Compressed,
Computation, ComputedVecFrom3, LazyVecFrom1, StoredIndex, StoredVec, Version,
AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Computation,
ComputedVecFrom3, Format, LazyVecFrom1, StoredIndex, StoredVec,
};
use crate::vecs::{Indexes, fetched, indexes};
@@ -33,7 +33,7 @@ pub struct ComputedValueVecsFromTxindex {
pub dollars: Option<ComputedVecsFromTxindex<Dollars>>,
}
const VERSION: Version = Version::ONE;
const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromTxindex {
#[allow(clippy::too_many_arguments)]
@@ -44,7 +44,7 @@ impl ComputedValueVecsFromTxindex {
source: Option<BoxedAnyIterableVec<TxIndex, Sats>>,
version: Version,
computation: Computation,
compressed: Compressed,
format: Format,
fetched: Option<&fetched::Vecs>,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
@@ -58,14 +58,14 @@ impl ComputedValueVecsFromTxindex {
path,
name,
compute_source,
VERSION + version,
compressed,
version + VERSION,
format,
options,
)?;
let bitcoin_txindex = LazyVecFrom1::init(
&name_in_btc,
VERSION + version,
version + VERSION,
source.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s),
|txindex: TxIndex, iter| {
iter.next_at(txindex.unwrap_to_usize()).map(|(_, value)| {
@@ -79,8 +79,8 @@ impl ComputedValueVecsFromTxindex {
path,
&name_in_btc,
false,
VERSION + version,
compressed,
version + VERSION,
format,
options,
)?;
@@ -89,8 +89,8 @@ impl ComputedValueVecsFromTxindex {
computation,
path,
&name_in_usd,
VERSION + version,
compressed,
version + VERSION,
format,
bitcoin_txindex.boxed_clone(),
indexes.txindex_to_height.boxed_clone(),
fetched.chainindexes_to_close.height.boxed_clone(),
@@ -125,8 +125,8 @@ impl ComputedValueVecsFromTxindex {
path,
&name_in_usd,
false,
VERSION + version,
compressed,
version + VERSION,
format,
options,
)
.unwrap()
@@ -203,7 +203,11 @@ impl ComputedValueVecsFromTxindex {
if let Some(dollars) = self.dollars.as_mut() {
let dollars_txindex = self.dollars_txindex.as_mut().unwrap();
dollars_txindex.compute_if_necessary(starting_indexes.txindex, exit)?;
dollars_txindex.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
dollars.compute_rest_from_bitcoin(
indexer,
@@ -0,0 +1,126 @@
use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
use crate::vecs::{Indexes, fetched, indexes};
#[derive(Clone)]
pub struct ComputedHeightValueVecs {
pub sats: Option<EagerVec<Height, Sats>>,
pub bitcoin: EagerVec<Height, Bitcoin>,
pub dollars: Option<EagerVec<Height, Dollars>>,
}
const VERSION: Version = Version::ZERO;
impl ComputedHeightValueVecs {
pub fn forced_import(
path: &Path,
name: &str,
compute_source: bool,
version: Version,
format: Format,
compute_dollars: bool,
) -> color_eyre::Result<Self> {
Ok(Self {
sats: compute_source.then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
.unwrap()
}),
bitcoin: EagerVec::forced_import(
path,
&format!("{name}_in_btc"),
version + VERSION + Version::ZERO,
format,
)?,
dollars: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&format!("{name}_in_usd"),
version + VERSION + Version::ZERO,
format,
)
.unwrap()
}),
})
}
pub fn compute_all<F>(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
where
F: FnMut(
&mut EagerVec<Height, Sats>,
&Indexer,
&indexes::Vecs,
&Indexes,
&Exit,
) -> Result<()>,
{
compute(
self.sats.as_mut().unwrap(),
indexer,
indexes,
starting_indexes,
exit,
)?;
let height: Option<&StoredVec<Height, Sats>> = None;
self.compute_rest(fetched, starting_indexes, exit, height)?;
Ok(())
}
pub fn compute_rest(
&mut self,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&impl CollectableVec<Height, Sats>>,
) -> color_eyre::Result<()> {
if let Some(height) = height {
self.bitcoin
.compute_from_sats(starting_indexes.height, height, exit)?;
} else {
self.bitcoin.compute_from_sats(
starting_indexes.height,
self.sats.as_ref().unwrap(),
exit,
)?;
}
let height_to_bitcoin = &self.bitcoin;
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_from_bitcoin(
starting_indexes.height,
height_to_bitcoin,
height_to_close,
exit,
)?;
}
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
vec![&self.bitcoin as &dyn AnyCollectableVec],
self.sats.as_ref().map_or(vec![], |v| vec![v]),
self.dollars.as_ref().map_or(vec![], |v| vec![v]),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+16 -11
View File
@@ -1,9 +1,9 @@
use std::{fs, path::Path};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, Computation, VecIterator, Version};
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
use super::{
Indexes,
@@ -11,6 +11,8 @@ use super::{
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
@@ -21,8 +23,9 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
version: Version,
_computation: Computation,
compressed: Compressed,
format: Format,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -31,22 +34,24 @@ impl Vecs {
path,
"difficulty",
false,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
path,
"difficultyepoch",
Version::ZERO,
compressed,
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
path,
"halvingepoch",
Version::ZERO,
compressed,
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
})
@@ -60,7 +65,7 @@ impl Vecs {
exit: &Exit,
) -> color_eyre::Result<()> {
let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
self.indexes_to_difficultyepoch.compute(
self.indexes_to_difficultyepoch.compute_all(
indexer,
indexes,
starting_indexes,
@@ -84,7 +89,7 @@ impl Vecs {
)?;
let mut height_to_halvingepoch_iter = indexes.height_to_halvingepoch.into_iter();
self.indexes_to_halvingepoch.compute(
self.indexes_to_halvingepoch.compute_all(
indexer,
indexes,
starting_indexes,
+69 -16
View File
@@ -1,9 +1,10 @@
use std::{fs, path::Path};
use brk_core::Version;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, Computation};
use brk_vec::{AnyCollectableVec, Computation, Format};
pub mod blocks;
pub mod constants;
@@ -12,10 +13,13 @@ pub mod grouped;
pub mod indexes;
pub mod market;
pub mod mining;
pub mod stateful;
pub mod transactions;
pub mod utxos;
pub use indexes::Indexes;
use log::info;
const VERSION: Version = Version::ONE;
#[derive(Clone)]
pub struct Vecs {
@@ -25,37 +29,78 @@ pub struct Vecs {
pub mining: mining::Vecs,
pub market: market::Vecs,
pub transactions: transactions::Vecs,
pub utxos: utxos::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
}
impl Vecs {
pub fn import(
path: &Path,
version: Version,
indexer: &Indexer,
fetch: bool,
computation: Computation,
compressed: Compressed,
format: Format,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let indexes = indexes::Vecs::forced_import(path, indexer, computation, compressed)?;
let indexes = indexes::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
indexer,
computation,
format,
)?;
let fetched =
fetch.then(|| fetched::Vecs::forced_import(path, computation, compressed).unwrap());
let fetched = fetch.then(|| {
fetched::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
)
.unwrap()
});
Ok(Self {
blocks: blocks::Vecs::forced_import(path, computation, compressed)?,
mining: mining::Vecs::forced_import(path, computation, compressed)?,
constants: constants::Vecs::forced_import(path, computation, compressed)?,
market: market::Vecs::forced_import(path, computation, compressed)?,
utxos: utxos::Vecs::forced_import(path, computation, compressed, fetched.as_ref())?,
blocks: blocks::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
)?,
mining: mining::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
)?,
constants: constants::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
)?,
market: market::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
)?,
stateful: stateful::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
fetched.as_ref(),
)?,
transactions: transactions::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
indexer,
&indexes,
computation,
compressed,
format,
fetched.as_ref(),
)?,
indexes,
@@ -72,16 +117,20 @@ impl Vecs {
) -> color_eyre::Result<()> {
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
self.constants
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing blocks...");
self.blocks
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing mining...");
self.mining
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(
indexer,
&self.indexes,
@@ -91,6 +140,7 @@ impl Vecs {
)?;
}
info!("Computing transactions...");
self.transactions.compute(
indexer,
&self.indexes,
@@ -100,6 +150,7 @@ impl Vecs {
)?;
if let Some(fetched) = self.fetched.as_ref() {
info!("Computing market...");
self.market.compute(
indexer,
&self.indexes,
@@ -110,12 +161,14 @@ impl Vecs {
)?;
}
self.utxos.compute(
info!("Computing stateful...");
self.stateful.compute(
indexer,
&self.indexes,
&self.transactions,
self.fetched.as_ref(),
&starting_indexes,
&self.market,
starting_indexes,
exit,
)?;
@@ -130,7 +183,7 @@ impl Vecs {
self.mining.vecs(),
self.market.vecs(),
self.transactions.vecs(),
self.utxos.vecs(),
self.stateful.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,258 @@
use std::{collections::BTreeMap, ops::ControlFlow};
use brk_core::{CheckedSub, Dollars, HalvingEpoch, Height, Result, Timestamp};
use brk_exit::Exit;
use brk_state::{BlockState, OutputFilter, Outputs, Transacted};
use brk_vec::StoredIndex;
use rayon::prelude::*;
use crate::vecs::Indexes;
use super::cohort;
pub trait OutputCohorts {
fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp);
fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]);
fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>);
fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>;
}
impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> {
fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
if chain_state.is_empty() {
return;
}
let prev_timestamp = chain_state.last().unwrap().timestamp;
self.by_date_range
.as_mut_vec()
.into_par_iter()
.for_each(|(filter, v)| {
let state = &mut v.state;
let _ = chain_state
.iter()
.try_for_each(|block_state| -> ControlFlow<()> {
let prev_days_old = block_state
.timestamp
.difference_in_days_between(prev_timestamp);
let days_old = block_state.timestamp.difference_in_days_between(timestamp);
if prev_days_old == days_old {
return ControlFlow::Continue(());
}
let is = filter.contains(days_old);
let was = filter.contains(prev_days_old);
if is && !was {
state.increment(&block_state.supply, block_state.price);
} else if was && !is {
state.decrement(&block_state.supply, block_state.price);
}
ControlFlow::Continue(())
});
});
}
fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]) {
let mut time_based_vecs = self
.by_date_range
.as_mut_vec()
.into_iter()
.chain(self.by_epoch.as_mut_vec())
.collect::<Vec<_>>();
let last_timestamp = chain_state.last().unwrap().timestamp;
let current_price = chain_state.last().unwrap().price;
// dbg!(&height_to_sent);
height_to_sent.into_iter().for_each(|(height, sent)| {
let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
let prev_price = block_state.price;
let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize();
let days_old = block_state
.timestamp
.difference_in_days_between(last_timestamp);
let days_old_foat = block_state
.timestamp
.difference_in_days_between_float(last_timestamp);
let older_than_hour =
jiff::Timestamp::from(last_timestamp.checked_sub(block_state.timestamp).unwrap())
.as_second()
>= 60 * 60;
time_based_vecs
.iter_mut()
.filter(|(filter, _)| match filter {
OutputFilter::From(from) => *from <= days_old,
OutputFilter::To(to) => *to > days_old,
OutputFilter::Range(range) => range.contains(&days_old),
OutputFilter::Epoch(epoch) => *epoch == HalvingEpoch::from(height),
_ => unreachable!(),
})
.for_each(|(_, vecs)| {
vecs.state.send(
&sent.spendable_supply,
current_price,
prev_price,
blocks_old,
days_old_foat,
older_than_hour,
);
});
sent.by_type.spendable.as_typed_vec().into_iter().for_each(
|(output_type, supply_state)| {
self.by_type.get_mut(output_type).1.state.send(
supply_state,
current_price,
prev_price,
blocks_old,
days_old_foat,
older_than_hour,
)
},
);
sent.by_size_group
.into_iter()
.for_each(|(group, supply_state)| {
self.by_size_range.get_mut(group).1.state.send(
&supply_state,
current_price,
prev_price,
blocks_old,
days_old_foat,
older_than_hour,
);
});
});
}
fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
let supply_state = received.spendable_supply;
[
&mut self.by_date_range.start_to_1d.1,
&mut self.by_epoch.mut_vec_from_height(height).1,
]
.into_iter()
.for_each(|v| {
v.state.receive(&supply_state, price);
});
self.by_type
.as_mut_vec()
.into_iter()
.for_each(|(filter, vecs)| {
let output_type = match filter {
OutputFilter::Type(output_type) => *output_type,
_ => unreachable!(),
};
vecs.state.receive(received.by_type.get(output_type), price)
});
received
.by_size_group
.into_iter()
.for_each(|(group, supply_state)| {
self.by_size_range
.get_mut(group)
.1
.state
.receive(&supply_state, price);
});
}
fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
let by_date_range = self.by_date_range.as_vec();
let by_size_range = self.by_size_range.as_vec();
[
vec![(&mut self.all.1, self.by_epoch.vecs().to_vec())],
self.by_from_date
.as_mut_vec()
.into_iter()
.map(|(filter, vecs)| {
(
vecs,
by_date_range
.into_iter()
.filter(|(other, _)| filter.includes(other))
.map(|(_, v)| v)
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
self.by_up_to_date
.as_mut_vec()
.into_iter()
.map(|(filter, vecs)| {
(
vecs,
by_date_range
.into_iter()
.filter(|(other, _)| filter.includes(other))
.map(|(_, v)| v)
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
self.by_term
.as_mut_vec()
.into_iter()
.map(|(filter, vecs)| {
(
vecs,
by_date_range
.into_iter()
.filter(|(other, _)| filter.includes(other))
.map(|(_, v)| v)
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
self.by_from_size
.as_mut_vec()
.into_iter()
.map(|(filter, vecs)| {
(
vecs,
by_size_range
.into_iter()
.filter(|(other, _)| filter.includes(other))
.map(|(_, v)| v)
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
self.by_up_to_size
.as_mut_vec()
.into_iter()
.map(|(filter, vecs)| {
(
vecs,
by_size_range
.into_iter()
.filter(|(other, _)| filter.includes(other))
.map(|(_, v)| v)
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
]
.into_par_iter()
.flatten()
.try_for_each(|(vecs, stateful)| {
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
})
}
}
+158 -129
View File
@@ -2,15 +2,13 @@ use std::{fs, path::Path};
use brk_core::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
StoredUsize, TxIndex, TxVersion, Weight,
StoredUsize, TxIndex, TxVersion, Version, Weight,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Compressed, Computation,
ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, StoredIndex, VecIterator,
Version,
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
};
use super::{
@@ -22,6 +20,8 @@ use super::{
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
@@ -89,10 +89,11 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
version: Version,
indexer: &Indexer,
indexes: &indexes::Vecs,
computation: Computation,
compressed: Compressed,
format: Format,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
@@ -103,8 +104,8 @@ impl Vecs {
computation,
path,
"value",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
indexer.vecs().inputindex_to_outputindex.boxed_clone(),
indexer.vecs().outputindex_to_value.boxed_clone(),
|index: InputIndex, inputindex_to_outputindex_iter, outputindex_to_value_iter| {
@@ -130,8 +131,8 @@ impl Vecs {
computation,
path,
"weight",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
indexer.vecs().txindex_to_base_size.boxed_clone(),
indexer.vecs().txindex_to_total_size.boxed_clone(),
|index: TxIndex, txindex_to_base_size_iter, txindex_to_total_size_iter| {
@@ -158,8 +159,8 @@ impl Vecs {
computation,
path,
"vsize",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
txindex_to_weight.boxed_clone(),
|index: TxIndex, iter| {
let index = index.unwrap_to_usize();
@@ -175,8 +176,8 @@ impl Vecs {
computation,
path,
"is_coinbase",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
indexes.txindex_to_height.boxed_clone(),
indexer.vecs().height_to_first_txindex.boxed_clone(),
|index: TxIndex, txindex_to_height_iter, height_to_first_txindex_iter| {
@@ -199,8 +200,8 @@ impl Vecs {
computation,
path,
"input_value",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
indexer.vecs().txindex_to_first_inputindex.boxed_clone(),
indexes.txindex_to_input_count.boxed_clone(),
inputindex_to_value.boxed_clone(),
@@ -236,20 +237,20 @@ impl Vecs {
// path,
// "input_value",
// true,
// Version::ZERO,
// compressed,
// version + VERSION + Version::ZERO,
// format,
// StorableVecGeneatorOptions::default()
// .add_average()
// .add_sum()
// .add_total(),
// .add_cumulative(),
// )?;
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation,
path,
"output_value",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
indexer.vecs().txindex_to_first_outputindex.boxed_clone(),
indexes.txindex_to_output_count.boxed_clone(),
indexer.vecs().outputindex_to_value.boxed_clone(),
@@ -285,20 +286,20 @@ impl Vecs {
// path,
// "output_value",
// true,
// Version::ZERO,
// compressed,
// version + VERSION + Version::ZERO,
// format,
// StorableVecGeneatorOptions::default()
// .add_average()
// .add_sum()
// .add_total(),
// .add_cumulative(),
// )?;
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
computation,
path,
"fee",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
txindex_to_input_value.boxed_clone(),
txindex_to_output_value.boxed_clone(),
|txindex: TxIndex, input_iter, output_iter| {
@@ -320,8 +321,8 @@ impl Vecs {
computation,
path,
"feerate",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
txindex_to_fee.boxed_clone(),
txindex_to_vsize.boxed_clone(),
|txindex: TxIndex, fee_iter, vsize_iter| {
@@ -341,77 +342,83 @@ impl Vecs {
path,
"tx_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
path,
"input_count",
false,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
path,
"output_count",
false,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
path,
"tx_v1",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
path,
"tx_v2",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
path,
"tx_v3",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
path,
"fee",
indexes,
Some(txindex_to_fee.boxed_clone()),
Version::ZERO,
version + VERSION + Version::ZERO,
computation,
compressed,
format,
fetched,
StorableVecGeneatorOptions::default()
.add_sum()
.add_total()
.add_cumulative()
.add_percentiles()
.add_minmax()
.add_average(),
@@ -420,8 +427,8 @@ impl Vecs {
path,
"feerate",
false,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_minmax()
@@ -431,8 +438,8 @@ impl Vecs {
path,
"tx_vsize",
false,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_minmax()
@@ -442,8 +449,8 @@ impl Vecs {
path,
"tx_weight",
false,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_minmax()
@@ -453,12 +460,12 @@ impl Vecs {
path,
"subsidy",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_sum()
.add_total()
.add_cumulative()
.add_minmax()
.add_average(),
compute_dollars,
@@ -467,11 +474,11 @@ impl Vecs {
path,
"coinbase",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_total()
.add_cumulative()
.add_percentiles()
.add_minmax()
.add_average(),
@@ -481,173 +488,175 @@ impl Vecs {
path,
"unclaimed_rewards",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
compute_dollars,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
path,
"p2a_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
path,
"p2ms_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
path,
"p2pk33_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
path,
"p2pk65_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
path,
"p2pkh_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
path,
"p2sh_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
path,
"p2tr_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
path,
"p2wpkh_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
path,
"p2wsh_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
path,
"opreturn_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
path,
"unknownoutput_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
path,
"emptyoutput_count",
true,
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
path,
"exact_utxo_count",
true,
Version::TWO,
compressed,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
txindex_to_is_coinbase,
@@ -728,20 +737,35 @@ impl Vecs {
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?;
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?;
self.txindex_to_is_coinbase
.compute_if_necessary(starting_indexes.txindex, exit)?;
self.txindex_to_is_coinbase.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
self.txindex_to_weight
.compute_if_necessary(starting_indexes.txindex, exit)?;
self.txindex_to_weight.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
self.txindex_to_vsize
.compute_if_necessary(starting_indexes.txindex, exit)?;
self.txindex_to_vsize.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
self.inputindex_to_value
.compute_if_necessary(starting_indexes.inputindex, exit)?;
self.inputindex_to_value.compute_if_necessary(
starting_indexes.inputindex,
&indexer.vecs().inputindex_to_outputindex,
exit,
)?;
self.txindex_to_output_value
.compute_if_necessary(starting_indexes.txindex, exit)?;
self.txindex_to_output_value.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
// self.indexes_to_output_value.compute_all(
// indexer,
@@ -759,8 +783,11 @@ impl Vecs {
// },
// )?;
self.txindex_to_input_value
.compute_if_necessary(starting_indexes.txindex, exit)?;
self.txindex_to_input_value.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
// self.indexes_to_input_value.compute_all(
// indexer,
@@ -778,6 +805,18 @@ impl Vecs {
// },
// )?;
self.txindex_to_fee.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
self.txindex_to_feerate.compute_if_necessary(
starting_indexes.txindex,
&indexer.vecs().txindex_to_txid,
exit,
)?;
self.indexes_to_fee.compute_rest(
indexer,
indexes,
@@ -855,12 +894,7 @@ impl Vecs {
self.indexes_to_fee.sats.height.unwrap_sum().iter();
vec.compute_transform(
starting_indexes.height,
self.indexes_to_coinbase
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
|(height, coinbase, ..)| {
let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
(height, coinbase.checked_sub(fees).unwrap())
@@ -879,12 +913,7 @@ impl Vecs {
|vec, _, _, starting_indexes, exit| {
vec.compute_transform(
starting_indexes.height,
self.indexes_to_subsidy
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
self.indexes_to_subsidy.sats.height.as_ref().unwrap(),
|(height, subsidy, ..)| {
let halving = HalvingEpoch::from(height);
let expected =
@@ -1085,16 +1114,16 @@ impl Vecs {
let mut input_count_iter = self
.indexes_to_input_count
.height
.unwrap_total()
.unwrap_cumulative()
.into_iter();
let mut opreturn_count_iter = self
.indexes_to_opreturn_count
.height_extra
.unwrap_total()
.unwrap_cumulative()
.into_iter();
v.compute_transform(
starting_indexes.height,
self.indexes_to_output_count.height.unwrap_total(),
self.indexes_to_output_count.height.unwrap_cumulative(),
|(h, output_count, ..)| {
let input_count = input_count_iter.unwrap_get_inner(h);
let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);
@@ -1,357 +0,0 @@
use std::{fs, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Sats, StoredUsize};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, Result, VecIterator, Version,
};
use crate::{
states::{CohortState, RealizedState},
vecs::{
Indexes, fetched,
grouped::{
ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
StorableVecGeneatorOptions,
},
indexes,
},
};
const VERSION: Version = Version::ZERO;
pub struct Vecs {
starting_height: Height,
pub state: CohortState,
pub height_to_realized_cap: Option<EagerVec<Height, Dollars>>,
pub indexes_to_realized_cap: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_supply: EagerVec<Height, Sats>,
pub indexes_to_supply: ComputedValueVecsFromHeight,
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_realized_price: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_realized_price_extra: Option<ComputedRatioVecsFromDateIndex>,
}
impl Vecs {
pub fn forced_import(
path: &Path,
cohort_name: Option<&str>,
_computation: Computation,
compressed: Compressed,
version: Version,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
fs::create_dir_all(path)?;
// let prefix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{s}_{name}"));
let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}"));
let mut state = CohortState::default();
if compute_dollars {
state.realized = Some(RealizedState::NAN);
}
Ok(Self {
starting_height: Height::ZERO,
state,
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("realized_cap"),
VERSION + Version::ZERO + version,
compressed,
)
.unwrap()
}),
indexes_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("realized_cap"),
false,
VERSION + Version::ZERO + version,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply: EagerVec::forced_import(
path,
&suffix("supply"),
VERSION + Version::ZERO + version,
compressed,
)?,
indexes_to_supply: ComputedValueVecsFromHeight::forced_import(
path,
&suffix("supply"),
false,
VERSION + Version::ZERO + version,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_utxo_count: EagerVec::forced_import(
path,
&suffix("utxo_count"),
VERSION + Version::ZERO + version,
compressed,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path,
&suffix("utxo_count"),
false,
VERSION + Version::ZERO + version,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
indexes_to_realized_price: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("realized_price"),
true,
VERSION + Version::new(2) + version,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
indexes_to_realized_price_extra: compute_dollars.then(|| {
ComputedRatioVecsFromDateIndex::forced_import(
path,
&suffix("realized_price"),
false,
VERSION + Version::new(2) + version,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
})
}
pub fn starting_height(&self) -> Height {
[
self.height_to_supply.len(),
self.height_to_utxo_count.len(),
self.height_to_realized_cap
.as_ref()
.map_or(usize::MAX, |v| v.len()),
]
.into_iter()
.map(Height::from)
.min()
.unwrap()
}
pub fn init(&mut self, starting_height: Height) {
if starting_height > self.starting_height() {
unreachable!()
}
self.starting_height = starting_height;
if let Some(prev_height) = starting_height.checked_sub(Height::new(1)) {
self.state.supply.value = self
.height_to_supply
.into_iter()
.unwrap_get_inner(prev_height);
self.state.supply.utxos = *self
.height_to_utxo_count
.into_iter()
.unwrap_get_inner(prev_height);
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
self.state.realized.as_mut().unwrap().realized_cap = height_to_realized_cap
.into_iter()
.unwrap_get_inner(prev_height);
}
}
}
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.height_to_supply
.validate_computed_version_or_reset_file(
base_version + self.height_to_supply.inner_version(),
)?;
self.height_to_utxo_count
.validate_computed_version_or_reset_file(
base_version + self.height_to_utxo_count.inner_version(),
)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() {
height_to_realized_cap.validate_computed_version_or_reset_file(
base_version + height_to_realized_cap.inner_version(),
)?;
}
Ok(())
}
pub fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> {
if self.starting_height > height {
return Ok(());
}
self.height_to_supply
.forced_push_at(height, self.state.supply.value, exit)?;
self.height_to_utxo_count.forced_push_at(
height,
StoredUsize::from(self.state.supply.utxos),
exit,
)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
height_to_realized_cap.forced_push_at(
height,
self.state
.realized
.as_ref()
.unwrap_or_else(|| {
dbg!(&self.state);
panic!();
})
.realized_cap,
exit,
)?;
}
Ok(())
}
pub fn safe_flush_height_vecs(&mut self, exit: &Exit) -> Result<()> {
self.height_to_supply.safe_flush(exit)?;
self.height_to_utxo_count.safe_flush(exit)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
height_to_realized_cap.safe_flush(exit)?;
}
Ok(())
}
pub fn compute_rest(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.indexes_to_supply.compute_rest(
indexer,
indexes,
fetched,
starting_indexes,
exit,
Some(&self.height_to_supply),
)?;
self.indexes_to_utxo_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_utxo_count),
)?;
if let Some(indexes_to_realized_cap) = self.indexes_to_realized_cap.as_mut() {
indexes_to_realized_cap.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_realized_cap.as_ref().unwrap()),
)?;
}
if let Some(indexes_to_realized_price) = self.indexes_to_realized_price.as_mut() {
indexes_to_realized_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_divide(
starting_indexes.height,
self.height_to_realized_cap.as_ref().unwrap(),
&**self.indexes_to_supply.bitcoin.height.as_ref().unwrap(),
exit,
)
},
)?;
}
if let Some(indexes_to_realized_price_extra) = self.indexes_to_realized_price_extra.as_mut()
{
indexes_to_realized_price_extra.compute_rest(
indexer,
indexes,
fetched.as_ref().unwrap(),
starting_indexes,
exit,
Some(
self.indexes_to_realized_price
.as_ref()
.unwrap()
.dateindex
.unwrap_last(),
),
)?;
}
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
vec![
&self.height_to_supply as &dyn AnyCollectableVec,
&self.height_to_utxo_count,
],
self.height_to_realized_cap
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_supply.vecs(),
self.indexes_to_utxo_count.vecs(),
self.indexes_to_realized_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_realized_price
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_realized_price_extra
.as_ref()
.map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
impl Clone for Vecs {
fn clone(&self) -> Self {
Self {
starting_height: self.starting_height,
state: CohortState::default(),
height_to_realized_cap: self.height_to_realized_cap.clone(),
indexes_to_realized_cap: self.indexes_to_realized_cap.clone(),
height_to_supply: self.height_to_supply.clone(),
indexes_to_supply: self.indexes_to_supply.clone(),
height_to_utxo_count: self.height_to_utxo_count.clone(),
indexes_to_utxo_count: self.indexes_to_utxo_count.clone(),
indexes_to_realized_price: self.indexes_to_realized_price.clone(),
indexes_to_realized_price_extra: self.indexes_to_realized_price_extra.clone(),
}
}
}
+5 -2
View File
@@ -4,20 +4,23 @@ description = "The Core (Structs and Errors) of the Bitcoin Research Kit"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
bincode = { workspace = true }
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
byteview = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
rapidhash = "1.4.0"
rlimit = "0.10.2"
serde = { workspace = true }
serde_derive = { workspace = true }
serde_bytes = "0.11.17"
serde_bytes = { workspace = true }
serde_json = { workspace = true }
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
@@ -1,20 +1,26 @@
use std::{
fmt::{self, Debug},
io,
time::SystemTimeError,
io, result, time,
};
use crate::Version;
pub type Result<T, E = Error> = std::result::Result<T, E>;
pub type Result<T, E = Error> = result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
IO(io::Error),
SerdeJson(serde_json::Error),
Jiff(jiff::Error),
Fjall(fjall::Error),
SystemTimeError(time::SystemTimeError),
ZeroCopyError,
BincodeEncodeError(bincode::error::EncodeError),
BincodeDecodeError(bincode::error::DecodeError),
WrongEndian,
DifferentVersion { found: Version, expected: Version },
MmapsVecIsTooSmall,
IO(io::Error),
ZeroCopyError,
IndexTooHigh,
EmptyVec,
IndexTooLow,
@@ -24,13 +30,16 @@ pub enum Error {
UnsupportedUnflushedState,
RangeFromAfterTo(usize, usize),
DifferentCompressionMode,
SystemTimeError,
ToSerdeJsonValueError(serde_json::Error),
WrongLength,
WrongAddressType,
UnindexableDate,
String(&'static str),
}
impl From<SystemTimeError> for Error {
fn from(_: SystemTimeError) -> Self {
Self::SystemTimeError
impl From<time::SystemTimeError> for Error {
fn from(value: time::SystemTimeError) -> Self {
Self::SystemTimeError(value)
}
}
@@ -40,6 +49,18 @@ impl From<io::Error> for Error {
}
}
impl From<jiff::Error> for Error {
fn from(value: jiff::Error) -> Self {
Self::Jiff(value)
}
}
impl From<fjall::Error> for Error {
fn from(value: fjall::Error) -> Self {
Self::Fjall(value)
}
}
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
@@ -54,13 +75,34 @@ impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
impl From<serde_json::Error> for Error {
fn from(error: serde_json::Error) -> Self {
Self::ToSerdeJsonValueError(error)
Self::SerdeJson(error)
}
}
impl From<bincode::error::DecodeError> for Error {
fn from(error: bincode::error::DecodeError) -> Self {
Self::BincodeDecodeError(error)
}
}
impl From<bincode::error::EncodeError> for Error {
fn from(error: bincode::error::EncodeError) -> Self {
Self::BincodeEncodeError(error)
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::IO(error) => Debug::fmt(&error, f),
Error::SystemTimeError(error) => Debug::fmt(&error, f),
Error::SerdeJson(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::Fjall(error) => Debug::fmt(&error, f),
Error::BincodeDecodeError(error) => Debug::fmt(&error, f),
Error::BincodeEncodeError(error) => Debug::fmt(&error, f),
Error::ZeroCopyError => write!(f, "ZeroCopy error"),
Error::WrongEndian => write!(f, "Wrong endian"),
Error::DifferentVersion { found, expected } => {
write!(
@@ -69,7 +111,6 @@ impl fmt::Display for Error {
)
}
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IO(error) => Debug::fmt(&error, f),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::IndexTooLow => write!(f, "Index too low"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
@@ -81,12 +122,17 @@ impl fmt::Display for Error {
"Unsupported unflush state, please flush before using this function"
)
}
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
Error::SystemTimeError => write!(f, "SystemTimeError"),
Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"),
Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
Error::ToSerdeJsonValueError(error) => Debug::fmt(&error, f),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(
f,
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
),
Error::String(s) => write!(f, "{s}"),
}
}
}
-49
View File
@@ -1,49 +0,0 @@
use std::{
fmt::{self, Debug},
io,
};
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
IO(io::Error),
Jiff(jiff::Error),
ZeroCopyError,
WrongLength,
WrongAddressType,
UnindexableDate,
}
impl From<io::Error> for Error {
fn from(value: io::Error) -> Self {
Self::IO(value)
}
}
impl From<jiff::Error> for Error {
fn from(value: jiff::Error) -> Self {
Self::Jiff(value)
}
}
impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
Self::ZeroCopyError
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::IO(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(f, "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"),
}
}
}
impl std::error::Error for Error {}
+2 -2
View File
@@ -1,9 +1,9 @@
#![doc = include_str!("../README.md")]
mod error;
mod enums;
mod structs;
mod utils;
pub use error::*;
pub use enums::*;
pub use structs::*;
pub use utils::*;
+2 -3
View File
@@ -63,9 +63,8 @@ impl From<AddressIndex> for usize {
}
}
impl TryFrom<ByteView> for AddressIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
impl From<ByteView> for AddressIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
@@ -15,9 +15,8 @@ pub struct AddressIndexOutputIndex {
outputindex: Outputindex,
}
impl TryFrom<ByteView> for AddressIndexOutputIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
impl From<ByteView> for AddressIndexOutputIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
@@ -5,8 +5,6 @@ use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
use super::{AddressBytes, OutputType};
#[derive(
@@ -41,10 +39,9 @@ impl From<[u8; 8]> for AddressBytesHash {
}
}
impl TryFrom<ByteView> for AddressBytesHash {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for AddressBytesHash {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
+23
View File
@@ -6,6 +6,8 @@ use std::{
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use super::{Sats, StoredF64};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
@@ -25,6 +27,21 @@ impl Mul for Bitcoin {
}
}
impl Mul<usize> for Bitcoin {
type Output = Self;
fn mul(self, rhs: usize) -> Self::Output {
Self::from(Sats::from(self) * rhs)
}
}
impl Div<Bitcoin> for Bitcoin {
type Output = StoredF64;
fn div(self, rhs: Bitcoin) -> Self::Output {
StoredF64::from(self.0 / rhs.0)
// Self::from(Sats::from(self) / Sats::from(rhs))
}
}
impl Div<usize> for Bitcoin {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
@@ -93,3 +110,9 @@ impl Ord for Bitcoin {
}
}
}
impl CheckedSub<usize> for Bitcoin {
fn checked_sub(self, rhs: usize) -> Option<Self> {
Some(Self(self.0 - rhs as f64))
}
}
@@ -3,7 +3,7 @@ use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Error, copy_first_8bytes};
use crate::copy_first_8bytes;
use super::BlockHash;
@@ -35,10 +35,9 @@ impl From<&BlockHash> for BlockHashPrefix {
}
}
impl TryFrom<ByteView> for BlockHashPrefix {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for BlockHashPrefix {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
+63 -10
View File
@@ -22,11 +22,17 @@ use super::Dollars;
KnownLayout,
Serialize,
)]
pub struct Cents(u64);
pub struct Cents(i64);
impl Cents {
pub const fn mint(value: i64) -> Self {
Self(value)
}
}
impl From<Dollars> for Cents {
fn from(value: Dollars) -> Self {
Self((*value * 100.0).round() as u64)
Self((*value * 100.0).round() as i64)
}
}
@@ -36,15 +42,45 @@ impl From<Cents> for f64 {
}
}
impl From<i64> for Cents {
fn from(value: i64) -> Self {
Self(value)
}
}
impl From<u64> for Cents {
fn from(value: u64) -> Self {
Self(value)
Self(value as i64)
}
}
impl From<Cents> for usize {
fn from(value: Cents) -> Self {
if value.0 < 0 {
panic!()
}
value.0 as usize
}
}
impl From<usize> for Cents {
fn from(value: usize) -> Self {
Self(value as i64)
}
}
impl From<Cents> for i64 {
fn from(value: Cents) -> Self {
value.0
}
}
impl From<Cents> for u64 {
fn from(value: Cents) -> Self {
value.0
if value.0 < 0 {
panic!("Shouldn't convert neg cents to u64")
}
value.0 as u64
}
}
@@ -55,24 +91,34 @@ impl Add for Cents {
}
}
impl Div<Cents> for Cents {
type Output = Self;
fn div(self, rhs: Self) -> Self::Output {
Self(self.0 / rhs.0)
}
}
impl Div<usize> for Cents {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
Self(self.0 / rhs as u64)
Self(self.0 / rhs as i64)
}
}
impl From<u128> for Cents {
fn from(value: u128) -> Self {
if value > u64::MAX as u128 {
panic!("u128 bigger than u64")
if value > i64::MAX as u128 {
panic!("u128 bigger than i64")
}
Self(value as u64)
Self(value as i64)
}
}
impl From<Cents> for u128 {
fn from(value: Cents) -> Self {
if value.0 < 0 {
panic!("Shouldn't convert neg cents to u128")
}
value.0 as u128
}
}
@@ -80,14 +126,21 @@ impl From<Cents> for u128 {
impl Mul<Cents> for Cents {
type Output = Cents;
fn mul(self, rhs: Cents) -> Self::Output {
Self(self.0 * rhs.0)
Self(self.0.checked_mul(rhs.0).unwrap())
}
}
impl Mul<i64> for Cents {
type Output = Cents;
fn mul(self, rhs: i64) -> Self::Output {
Self(self.0 * rhs)
}
}
impl Mul<usize> for Cents {
type Output = Cents;
fn mul(self, rhs: usize) -> Self::Output {
Self(self.0 * rhs as u64)
Self(self.0 * rhs as i64)
}
}
+17 -1
View File
@@ -1,4 +1,7 @@
use std::ops::Add;
use std::{
fmt,
ops::{Add, Rem},
};
use serde::Serialize;
// use color_eyre::eyre::eyre;
@@ -77,3 +80,16 @@ impl CheckedSub for DateIndex {
self.0.checked_sub(rhs.0).map(Self)
}
}
impl Rem<usize> for DateIndex {
type Output = Self;
fn rem(self, rhs: usize) -> Self::Output {
Self(self.0 % rhs as u16)
}
}
impl fmt::Display for DateIndex {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0)
}
}
+116 -21
View File
@@ -4,16 +4,30 @@ use std::{
ops::{Add, AddAssign, Div, Mul},
};
use bincode::{Decode, Encode};
use byteview::ByteView;
use derive_deref::Deref;
use serde::Serialize;
use serde::{Deserialize, Serialize};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use crate::{CheckedSub, copy_first_8bytes};
use super::{Bitcoin, Cents, Close, Sats, StoredF32, StoredF64};
use super::{Bitcoin, Cents, Close, High, Sats, StoredF32, StoredF64};
#[derive(
Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
Debug,
Default,
Clone,
Copy,
Deref,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
Deserialize,
Encode,
Decode,
)]
pub struct Dollars(f64);
@@ -62,6 +76,12 @@ impl From<Close<Dollars>> for Dollars {
}
}
impl From<High<Dollars>> for Dollars {
fn from(value: High<Dollars>) -> Self {
Self(value.0)
}
}
impl From<usize> for Dollars {
fn from(value: usize) -> Self {
Self(value as f64)
@@ -78,10 +98,10 @@ impl Add for Dollars {
impl Div<Dollars> for Dollars {
type Output = StoredF64;
fn div(self, rhs: Dollars) -> Self::Output {
if self.is_nan() {
StoredF64::from(self.0)
if self.is_nan() || rhs == Dollars::ZERO {
StoredF64::NAN
} else {
StoredF64::from(self.0 / rhs.0)
StoredF64::from(f64::from(self) / f64::from(rhs))
}
}
}
@@ -89,10 +109,10 @@ impl Div<Dollars> for Dollars {
impl Div<Close<Dollars>> for Dollars {
type Output = StoredF64;
fn div(self, rhs: Close<Dollars>) -> Self::Output {
if self.is_nan() {
StoredF64::from(self.0)
if self.is_nan() || *rhs == Dollars::ZERO {
StoredF64::NAN
} else {
StoredF64::from(self.0 / rhs.0)
StoredF64::from(f64::from(self) / f64::from(*rhs))
}
}
}
@@ -100,10 +120,10 @@ impl Div<Close<Dollars>> for Dollars {
impl Div<Dollars> for Close<Dollars> {
type Output = StoredF64;
fn div(self, rhs: Dollars) -> Self::Output {
if self.is_nan() {
StoredF64::from(self.0)
if self.is_nan() || rhs == Dollars::ZERO {
StoredF64::NAN
} else {
StoredF64::from(self.0 / rhs.0)
StoredF64::from(f64::from(*self) / f64::from(rhs))
}
}
}
@@ -111,8 +131,8 @@ impl Div<Dollars> for Close<Dollars> {
impl Div<usize> for Dollars {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
if self.is_nan() {
self
if self.is_nan() || rhs == 0 {
Dollars::NAN
} else {
Self::from(Cents::from(self) / rhs)
}
@@ -130,15 +150,56 @@ impl Div<Bitcoin> for Dollars {
}
}
impl Mul<Dollars> for Dollars {
type Output = Self;
fn mul(self, rhs: Dollars) -> Self::Output {
Self::from(Cents::from(self) * Cents::from(rhs))
}
}
impl Mul<Close<Dollars>> for Dollars {
type Output = Self;
fn mul(self, rhs: Close<Dollars>) -> Self::Output {
Self::from(Cents::from(self) * Cents::from(*rhs))
}
}
impl Mul<Dollars> for Close<Dollars> {
type Output = Dollars;
fn mul(self, rhs: Dollars) -> Self::Output {
Dollars::from(Cents::from(*self) * Cents::from(rhs))
}
}
impl Mul<usize> for Close<Dollars> {
type Output = Dollars;
fn mul(self, rhs: usize) -> Self::Output {
Dollars::from(Cents::from(*self) * rhs)
}
}
impl Mul<Bitcoin> for Dollars {
type Output = Self;
fn mul(self, rhs: Bitcoin) -> Self::Output {
self * Sats::from(rhs)
}
}
impl Mul<Bitcoin> for Close<Dollars> {
type Output = Dollars;
fn mul(self, rhs: Bitcoin) -> Self::Output {
*self * Sats::from(rhs)
}
}
impl Mul<Sats> for Dollars {
type Output = Self;
fn mul(self, rhs: Sats) -> Self::Output {
if self.is_nan() {
self
} else {
Self::from(Cents::from(
u128::from(Sats::from(rhs)) * u128::from(Cents::from(self))
/ u128::from(Sats::ONE_BTC),
u128::from(rhs) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC),
))
}
}
@@ -147,14 +208,21 @@ impl Mul<Bitcoin> for Dollars {
impl Mul<StoredF32> for Dollars {
type Output = Self;
fn mul(self, rhs: StoredF32) -> Self::Output {
if rhs.is_nan() {
self
if rhs.fract() != 0.0 {
Self::from(self.0 * *rhs as f64)
} else {
Self::from(Cents::from(Self::from(self.0 * *rhs as f64)))
self * *rhs as i64
}
}
}
impl Mul<i64> for Dollars {
type Output = Self;
fn mul(self, rhs: i64) -> Self::Output {
Self::from(Cents::from(self) * rhs)
}
}
impl Mul<usize> for Dollars {
type Output = Self;
fn mul(self, rhs: usize) -> Self::Output {
@@ -172,6 +240,12 @@ impl From<u128> for Dollars {
}
}
impl From<StoredF64> for Dollars {
fn from(value: StoredF64) -> Self {
Self(*value)
}
}
impl From<Close<Dollars>> for u128 {
fn from(value: Close<Dollars>) -> Self {
u128::from(*value)
@@ -204,7 +278,9 @@ impl CheckedSub for Dollars {
impl CheckedSub<usize> for Dollars {
fn checked_sub(self, rhs: usize) -> Option<Self> {
Some(Self(self.0 - rhs as f64))
Some(Dollars::from(
Cents::from(self).checked_sub(Cents::from(rhs)).unwrap(),
))
}
}
@@ -239,3 +315,22 @@ impl Ord for Dollars {
}
}
}
impl From<ByteView> for Dollars {
fn from(value: ByteView) -> Self {
let bytes = copy_first_8bytes(&value).unwrap();
Self::from(f64::from_be_bytes(bytes))
}
}
impl From<Dollars> for ByteView {
fn from(value: Dollars) -> Self {
Self::from(&value)
}
}
impl From<&Dollars> for ByteView {
fn from(value: &Dollars) -> Self {
Self::new(&value.to_be_bytes())
}
}
+12 -4
View File
@@ -4,12 +4,15 @@ use std::{
};
use bitcoincore_rpc::{Client, RpcApi};
use byteview::ByteView;
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use super::StoredUsize;
#[derive(
Debug,
Clone,
@@ -147,11 +150,17 @@ impl From<u64> for Height {
}
}
impl From<StoredUsize> for Height {
fn from(value: StoredUsize) -> Self {
Self(*value as u32)
}
}
impl From<usize> for Height {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Height> for usize {
fn from(value: Height) -> Self {
value.0 as usize
@@ -189,10 +198,9 @@ impl TryFrom<&std::path::Path> for Height {
}
}
impl TryFrom<byteview::ByteView> for Height {
type Error = crate::Error;
fn try_from(value: byteview::ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for Height {
fn from(value: byteview::ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
+2
View File
@@ -36,6 +36,7 @@ mod txidprefix;
mod txindex;
mod txversion;
mod unit;
mod version;
mod vin;
mod vout;
mod weekindex;
@@ -80,6 +81,7 @@ pub use txidprefix::*;
pub use txindex::*;
pub use txversion::*;
pub use unit::*;
pub use version::*;
pub use vin::*;
pub use vout::*;
pub use weekindex::*;
@@ -6,7 +6,7 @@ use serde::Serialize;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Error};
use crate::CheckedSub;
#[derive(
Debug,
@@ -82,10 +82,9 @@ impl Add<OutputTypeIndex> for OutputTypeIndex {
Self(self.0 + rhs.0)
}
}
impl TryFrom<ByteView> for OutputTypeIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for OutputTypeIndex {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
impl From<OutputTypeIndex> for ByteView {
+60 -5
View File
@@ -3,11 +3,13 @@ use std::{
ops::{Add, AddAssign, Div, Mul, SubAssign},
};
use bincode::{Decode, Encode};
use bitcoin::Amount;
use serde::Serialize;
use byteview::ByteView;
use serde::{Deserialize, Serialize};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use crate::{CheckedSub, copy_first_8bytes};
use super::{Bitcoin, Cents, Dollars, Height};
@@ -25,6 +27,9 @@ use super::{Bitcoin, Cents, Dollars, Height};
IntoBytes,
KnownLayout,
Serialize,
Deserialize,
Encode,
Decode,
)]
pub struct Sats(u64);
@@ -67,6 +72,12 @@ impl CheckedSub<Sats> for Sats {
}
}
impl CheckedSub<usize> for Sats {
fn checked_sub(self, rhs: usize) -> Option<Self> {
self.0.checked_sub(rhs as u64).map(Self::from)
}
}
impl SubAssign for Sats {
fn sub_assign(&mut self, rhs: Self) {
*self = self.checked_sub(rhs).unwrap();
@@ -76,21 +87,28 @@ impl SubAssign for Sats {
impl Mul<Sats> for Sats {
type Output = Self;
fn mul(self, rhs: Sats) -> Self::Output {
Sats::from(self.0 * rhs.0)
Sats::from(self.0.checked_mul(rhs.0).unwrap())
}
}
impl Mul<usize> for Sats {
type Output = Self;
fn mul(self, rhs: usize) -> Self::Output {
Sats::from(self.0.checked_mul(rhs as u64).unwrap())
}
}
impl Mul<u64> for Sats {
type Output = Self;
fn mul(self, rhs: u64) -> Self::Output {
Sats::from(self.0 * rhs)
Sats::from(self.0.checked_mul(rhs).unwrap())
}
}
impl Mul<Height> for Sats {
type Output = Self;
fn mul(self, rhs: Height) -> Self::Output {
Sats::from(self.0 * u64::from(rhs))
Sats::from(self.0.checked_mul(u64::from(rhs)).unwrap())
}
}
@@ -113,6 +131,17 @@ impl Div<Dollars> for Sats {
}
}
impl Div<Sats> for Sats {
type Output = Self;
fn div(self, rhs: Sats) -> Self::Output {
if rhs.0 == 0 {
Self(0)
} else {
Self(self.0 / rhs.0)
}
}
}
impl Div<usize> for Sats {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
@@ -187,3 +216,29 @@ impl From<Sats> for u128 {
value.0 as u128
}
}
impl From<ByteView> for Sats {
fn from(value: ByteView) -> Self {
let bytes = copy_first_8bytes(&value).unwrap();
Self::from(u64::from_be_bytes(bytes))
}
}
impl From<&Sats> for ByteView {
fn from(value: &Sats) -> Self {
Self::new(&value.0.to_be_bytes())
}
}
impl From<Sats> for ByteView {
fn from(value: Sats) -> Self {
Self::from(&value)
}
}
impl Mul<Sats> for usize {
type Output = Sats;
fn mul(self, rhs: Sats) -> Self::Output {
Self::Output::from(rhs.0 * self as u64)
}
}
+16
View File
@@ -1,3 +1,4 @@
use core::panic;
use std::{
cmp::Ordering,
ops::{Add, Div, Mul, Sub},
@@ -24,10 +25,19 @@ impl From<f32> for StoredF32 {
impl From<f64> for StoredF32 {
fn from(value: f64) -> Self {
if value > f32::MAX as f64 {
panic!("f64 is too big")
}
Self(value as f32)
}
}
impl From<StoredF32> for f64 {
fn from(value: StoredF32) -> Self {
value.0 as f64
}
}
impl From<StoredF64> for StoredF32 {
fn from(value: StoredF64) -> Self {
Self(*value as f32)
@@ -66,6 +76,12 @@ impl From<StoredF32> for f32 {
}
}
impl From<Dollars> for StoredF32 {
fn from(value: Dollars) -> Self {
StoredF32::from(f64::from(value))
}
}
impl Div<Dollars> for StoredF32 {
type Output = Self;
fn div(self, rhs: Dollars) -> Self::Output {
+18 -1
View File
@@ -1,5 +1,6 @@
use std::{
cmp::Ordering,
f64,
ops::{Add, Div, Mul},
};
@@ -7,13 +8,17 @@ use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use crate::{Bitcoin, CheckedSub, Dollars};
#[derive(
Debug, Deref, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
)]
pub struct StoredF64(f64);
impl StoredF64 {
pub const NAN: Self = Self(f64::NAN);
}
impl From<f64> for StoredF64 {
fn from(value: f64) -> Self {
Self(value)
@@ -59,6 +64,12 @@ impl From<StoredF64> for f64 {
}
}
impl From<Dollars> for StoredF64 {
fn from(value: Dollars) -> Self {
Self(f64::from(value))
}
}
impl CheckedSub<usize> for StoredF64 {
fn checked_sub(self, rhs: usize) -> Option<Self> {
Some(Self(self.0 - rhs as f64))
@@ -96,3 +107,9 @@ impl Ord for StoredF64 {
}
}
}
impl From<Bitcoin> for StoredF64 {
fn from(value: Bitcoin) -> Self {
Self(f64::from(value))
}
}
+13
View File
@@ -62,6 +62,19 @@ impl Timestamp {
}
}
}
pub fn difference_in_days_between_float(&self, other: Self) -> f64 {
match self.cmp(&other) {
Ordering::Equal => 0.0,
Ordering::Greater => other.difference_in_days_between_float(*self),
Ordering::Less => {
jiff::Timestamp::from(*self)
.duration_until(jiff::Timestamp::from(other))
.as_secs() as f64
/ ONE_DAY_IN_SEC as f64
}
}
}
}
impl From<u32> for Timestamp {
+4 -5
View File
@@ -3,7 +3,7 @@ use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Error, copy_first_8bytes};
use crate::copy_first_8bytes;
use super::Txid;
@@ -35,10 +35,9 @@ impl From<&Txid> for TxidPrefix {
}
}
impl TryFrom<ByteView> for TxidPrefix {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for TxidPrefix {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
+4 -5
View File
@@ -6,7 +6,7 @@ use serde::Serialize;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Error};
use crate::CheckedSub;
use super::StoredU32;
@@ -95,10 +95,9 @@ impl From<TxIndex> for usize {
}
}
impl TryFrom<ByteView> for TxIndex {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
Ok(Self::read_from_bytes(&value)?)
impl From<ByteView> for TxIndex {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
impl From<TxIndex> for ByteView {
@@ -1,6 +1,7 @@
use std::{
fs,
io::{self, Read},
iter::Sum,
ops::Add,
path::Path,
};
@@ -13,14 +14,14 @@ use crate::{Error, Result};
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout,
)]
pub struct Version(u32);
pub struct Version(u64);
impl Version {
pub const ZERO: Self = Self(0);
pub const ONE: Self = Self(1);
pub const TWO: Self = Self(2);
pub const fn new(v: u32) -> Self {
pub const fn new(v: u64) -> Self {
Self(v)
}
@@ -49,8 +50,8 @@ impl Version {
}
}
impl From<u32> for Version {
fn from(value: u32) -> Self {
impl From<u64> for Version {
fn from(value: u64) -> Self {
Self(value)
}
}
@@ -58,7 +59,7 @@ impl From<u32> for Version {
impl TryFrom<&Path> for Version {
type Error = Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
let mut buf = [0; 4];
let mut buf = [0; 8];
fs::read(value)?.as_slice().read_exact(&mut buf)?;
Ok(*(Self::ref_from_bytes(&buf)?))
}
@@ -70,3 +71,9 @@ impl Add<Version> for Version {
Self(self.0 + rhs.0)
}
}
impl Sum for Version {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
iter.fold(Self::ZERO, Add::add)
}
}
+4 -2
View File
@@ -1,9 +1,11 @@
use crate::{Error, Result};
#[allow(clippy::result_unit_err)]
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8], ()> {
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
return Err(());
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
+1
View File
@@ -4,6 +4,7 @@ description = "An exit blocker built on top of ctrlc"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
+4
View File
@@ -57,6 +57,10 @@ impl Exit {
self.blocking.store(true, Ordering::SeqCst);
}
pub fn blocked(&self) -> bool {
self.blocking.load(Ordering::SeqCst)
}
pub fn release(&self) {
self.blocking.store(false, Ordering::SeqCst);
}
+1
View File
@@ -4,6 +4,7 @@ description = "A Bitcoin price fetcher"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
+1 -4
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
@@ -34,4 +31,4 @@
</a>
</p>
A crate that can fetch the Bitcoin price, either by date or height, from multiple APIs such Kraken, Binance and Kibo.money.
A crate that can fetch the Bitcoin price, either by date or height, from Binance and Kibo.
+10 -6
View File
@@ -1,18 +1,22 @@
use brk_core::Date;
use brk_fetcher::Fetcher;
use brk_core::{Date, Height};
use brk_fetcher::{BRK, Fetcher};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
brk_logger::init(None);
let mut brk = BRK::default();
dbg!(brk.get_from_height(Height::new(900_000))?);
dbg!(brk.get_from_date(Date::new(2025, 6, 7))?);
let mut fetcher = Fetcher::import(None)?;
dbg!(fetcher.get_date(Date::new(2025, 1, 1))?);
dbg!(fetcher.get_date(Date::new(2025, 6, 5))?);
dbg!(fetcher.get_height(
881_000_u32.into(),
1740683986_u32.into(),
Some(1740683000_u32.into())
899911_u32.into(),
1749133056_u32.into(),
Some(1749132055_u32.into())
)?);
Ok(())
+2 -2
View File
@@ -17,7 +17,7 @@ use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, fetchers::retry};
pub struct Binance {
path: Option<PathBuf>,
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
_1d: Option<BTreeMap<Date, OHLCCents>>,
pub _1d: Option<BTreeMap<Date, OHLCCents>>,
har: Option<BTreeMap<Timestamp, OHLCCents>>,
}
@@ -95,7 +95,7 @@ impl Binance {
}
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Kraken...");
info!("Fetching daily prices from Binance...");
retry(
|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?),
+143
View File
@@ -0,0 +1,143 @@
use std::collections::BTreeMap;
use brk_core::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
use color_eyre::eyre::{ContextCompat, eyre};
use log::info;
use serde_json::Value;
use crate::{Close, Dollars, High, Low, Open, fetchers::retry};
#[derive(Default, Clone)]
pub struct BRK {
height_to_ohlc: BTreeMap<Height, Vec<OHLCCents>>,
dateindex_to_ohlc: BTreeMap<DateIndex, Vec<OHLCCents>>,
}
const API_URL: &str = "https://bitcoinresearchkit.org/api";
const RETRIES: usize = 10;
const CHUNK_SIZE: usize = 10_000;
impl BRK {
pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
let key = height.checked_sub(height % CHUNK_SIZE).unwrap();
#[allow(clippy::map_entry)]
if !self.height_to_ohlc.contains_key(&key)
|| ((key + self.height_to_ohlc.get(&key).unwrap().len()) <= height)
{
self.height_to_ohlc.insert(
key,
Self::fetch_height_prices(key).inspect_err(|e| {
dbg!(e);
})?,
);
}
self.height_to_ohlc
.get(&key)
.unwrap()
.get(usize::from(height.checked_sub(key).unwrap()))
.cloned()
.ok_or(eyre!("Couldn't find height in kibo"))
}
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
info!("Fetching Kibo height {height} prices...");
retry(
|_| {
let url = format!(
"{API_URL}/query?index=height&values=ohlc&from={}&to={}",
height,
height + CHUNK_SIZE
);
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.context("Expect to be an array")?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
},
30,
RETRIES,
)
}
pub fn get_from_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
let dateindex = DateIndex::try_from(date)?;
let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap();
#[allow(clippy::map_entry)]
if !self.dateindex_to_ohlc.contains_key(&key)
|| ((key + self.dateindex_to_ohlc.get(&key).unwrap().len()) <= dateindex)
{
self.dateindex_to_ohlc.insert(
key,
Self::fetch_date_prices(key).inspect_err(|e| {
dbg!(e);
})?,
);
}
self.dateindex_to_ohlc
.get(&key)
.unwrap()
.get(usize::from(dateindex.checked_sub(key).unwrap()))
.cloned()
.ok_or(eyre!("Couldn't find date in kibo"))
}
fn fetch_date_prices(dateindex: DateIndex) -> color_eyre::Result<Vec<OHLCCents>> {
info!("Fetching Kibo dateindex {dateindex} prices...");
retry(
|_| {
let url = format!(
"{API_URL}/query?index=dateindex&values=ohlc&from={}&to={}",
dateindex,
dateindex + CHUNK_SIZE
);
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.context("Expect to be an array")?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
},
30,
RETRIES,
)
}
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
let ohlc = value.as_array().context("Expect as_array to work")?;
let get_value = |index: usize| -> color_eyre::Result<_> {
Ok(Cents::from(Dollars::from(
ohlc.get(index)
.context("Expect index key to work")?
.as_f64()
.context("Expect as_f64 to work")?,
)))
};
Ok(OHLCCents::from((
Open::new(get_value(0)?),
High::new(get_value(1)?),
Low::new(get_value(2)?),
Close::new(get_value(3)?),
)))
}
pub fn clear(&mut self) {
self.height_to_ohlc.clear();
self.dateindex_to_ohlc.clear();
}
}
-155
View File
@@ -1,155 +0,0 @@
use std::{collections::BTreeMap, str::FromStr};
use brk_core::{CheckedSub, Date, Height, OHLCCents};
use color_eyre::eyre::{ContextCompat, eyre};
use log::info;
use serde_json::Value;
use crate::{Cents, Close, Dollars, High, Low, Open, fetchers::retry};
#[derive(Default, Clone)]
pub struct Kibo {
height_to_ohlc_vec: BTreeMap<Height, Vec<OHLCCents>>,
year_to_date_to_ohlc: BTreeMap<u16, BTreeMap<Date, OHLCCents>>,
}
const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
const RETRIES: usize = 10;
impl Kibo {
pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
let key = height.checked_sub(height % 10_000).unwrap_or_default();
#[allow(clippy::map_entry)]
if !self.height_to_ohlc_vec.contains_key(&key)
|| ((key + self.height_to_ohlc_vec.get(&key).unwrap().len()) <= height)
{
self.height_to_ohlc_vec.insert(
key,
Self::fetch_height_prices(key).inspect_err(|e| {
dbg!(e);
})?,
);
}
self.height_to_ohlc_vec
.get(&key)
.unwrap()
.get(usize::from(height.checked_sub(key).unwrap()))
.cloned()
.ok_or(eyre!("Couldn't find height in kibo"))
}
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
info!("Fetching Kibo height {height} prices...");
retry(
|_| {
let url = format!("{KIBO_OFFICIAL_URL}/height-to-price?chunk={}", height);
let body: Value = minreq::get(url).send()?.json()?;
body.as_object()
.context("Expect to be an object")?
.get("dataset")
.context("Expect object to have dataset")?
.as_object()
.context("Expect to be an object")?
.get("map")
.context("Expect to have map")?
.as_array()
.context("Expect to be an array")?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
},
30,
RETRIES,
)
}
pub fn get_from_date(&mut self, date: &Date) -> color_eyre::Result<OHLCCents> {
let year = date.year();
#[allow(clippy::map_entry)]
if !self.year_to_date_to_ohlc.contains_key(&year)
|| self
.year_to_date_to_ohlc
.get(&year)
.unwrap()
.last_key_value()
.unwrap()
.0
<= date
{
self.year_to_date_to_ohlc
.insert(year, Self::fetch_date_prices(year)?);
}
self.year_to_date_to_ohlc
.get(&year)
.unwrap()
.get(date)
.cloned()
.ok_or(eyre!("Couldn't find date in kibo"))
}
fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching Kibo date {year} prices...");
retry(
|_| {
let body: Value =
minreq::get(format!("{KIBO_OFFICIAL_URL}/date-to-price?chunk={}", year))
.send()?
.json()?;
body.as_object()
.context("Expect to be an object")?
.get("dataset")
.context("Expect object to have dataset")?
.as_object()
.context("Expect to be an object")?
.get("map")
.context("Expect to have map")?
.as_object()
.context("Expect to be an object")?
.iter()
.map(|(serialized_date, value)| -> color_eyre::Result<_> {
let date =
Date::from(jiff::civil::Date::from_str(serialized_date).unwrap());
Ok((date, Self::value_to_ohlc(value)?))
})
.collect::<Result<BTreeMap<_, _>, _>>()
},
30,
RETRIES,
)
}
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
let ohlc = value.as_object().context("Expect as_object to work")?;
let get_value = |key: &str| -> color_eyre::Result<_> {
Ok(Cents::from(Dollars::from(
ohlc.get(key)
.context("Expect get key to work")?
.as_f64()
.context("Expect as_f64 to work")?,
)))
};
Ok(OHLCCents::from((
Open::new(get_value("open")?),
High::new(get_value("high")?),
Low::new(get_value("low")?),
Close::new(get_value("close")?),
)))
}
pub fn clear(&mut self) {
self.height_to_ohlc_vec.clear();
self.year_to_date_to_ohlc.clear();
}
}
+2 -2
View File
@@ -1,9 +1,9 @@
mod binance;
mod kibo;
mod brk;
mod kraken;
mod retry;
pub use binance::*;
pub use kibo::*;
pub use brk::*;
pub use kraken::*;
use retry::*;
+45 -20
View File
@@ -5,18 +5,21 @@
use std::{collections::BTreeMap, fs, path::Path, thread::sleep, time::Duration};
use brk_core::{Cents, Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
use brk_core::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
use color_eyre::eyre::Error;
mod fetchers;
use fetchers::*;
pub use fetchers::*;
use log::info;
const TRIES: usize = 12 * 60;
#[derive(Clone)]
pub struct Fetcher {
binance: Binance,
kraken: Kraken,
kibo: Kibo,
brk: BRK,
}
impl Fetcher {
@@ -28,20 +31,37 @@ impl Fetcher {
Ok(Self {
binance: Binance::init(hars_path),
kraken: Kraken::default(),
kibo: Kibo::default(),
brk: BRK::default(),
})
}
pub fn get_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
self.kraken
self.get_date_(date, 0)
}
fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
self.binance
.get_from_1d(&date)
.or_else(|_| {
// eprintln!("{e}");
self.binance.get_from_1d(&date)
self.kraken.get_from_1d(&date)
})
.or_else(|_| {
// eprintln!("{e}");
self.brk.get_from_date(date)
})
.or_else(|e| {
eprintln!("{e}");
self.kibo.get_from_date(&date)
sleep(Duration::from_secs(60));
if tries < TRIES {
self.clear();
// dbg!(e, date, &self.binance._1d);
info!("Retrying to fetch date price...");
self.get_date_(date, tries + 1)
} else {
info!("Failed to fetch date prices...");
Err(e)
}
})
}
@@ -70,25 +90,32 @@ impl Fetcher {
let previous_timestamp = previous_timestamp.map(|t| t.floor_seconds());
let ohlc = self
.kraken
.binance
.get_from_1mn(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
// eprintln!("{e}");
self.binance
.unwrap_or_else(|_report| {
// eprintln!("{_report}");
self.kraken
.get_from_1mn(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
// eprintln!("{e}");
self.kibo.get_from_height(height).unwrap_or_else(|_| {
sleep(Duration::from_secs(30));
.unwrap_or_else(|_report| {
// // eprintln!("{_report}");
self.brk.get_from_height(height).unwrap_or_else(|_report| {
// eprintln!("{_report}");
if tries < 8 * 60 * 2 {
sleep(Duration::from_secs(60));
if tries < TRIES {
self.clear();
info!("Retrying to fetch height prices...");
// dbg!((height, timestamp, previous_timestamp));
return self
.get_height_(height, timestamp, previous_timestamp, tries + 1)
.unwrap();
}
info!("Failed to fetch height prices");
let date = Date::from(timestamp);
// eprintln!("{e}");
panic!(
@@ -112,8 +139,6 @@ How to fix this:
})
});
// self.ohlc.height.insert(height, ohlc);
Ok(ohlc)
}
@@ -161,7 +186,7 @@ How to fix this:
pub fn clear(&mut self) {
self.binance.clear();
self.kibo.clear();
self.brk.clear();
self.kraken.clear();
}
}
+3 -3
View File
@@ -4,6 +4,7 @@ description = "A Bitcoin Core indexer built on top of brk_parser"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
@@ -11,12 +12,11 @@ bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_parser = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vec = { workspace = true }
byteview = { workspace = true }
color-eyre = { workspace = true }
fjall = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
zerocopy = { workspace = true }
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
+5 -4
View File
@@ -3,7 +3,8 @@ use std::{path::Path, time::Instant};
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::{Parser, rpc};
use brk_parser::Parser;
use brk_vec::Format;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -14,9 +15,9 @@ fn main() -> color_eyre::Result<()> {
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
@@ -24,7 +25,7 @@ fn main() -> color_eyre::Result<()> {
let outputs = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs, false, false)?;
let mut indexer = Indexer::new(outputs, Format::Raw, false)?;
indexer.import_stores()?;
indexer.import_vecs()?;
+2 -2
View File
@@ -2,7 +2,7 @@ use bitcoincore_rpc::Client;
use brk_core::{
BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, OutputTypeIndex, P2AIndex, P2MSIndex, P2PK33Index, P2PK65Index, P2PKHIndex,
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, Result, TxIndex, UnknownOutputIndex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::{AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType};
@@ -48,7 +48,7 @@ impl Indexes {
}
}
pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> {
pub fn push_if_needed(&self, vecs: &mut Vecs) -> Result<()> {
let height = self.height;
vecs.height_to_first_txindex
.push_if_needed(height, self.txindex)?;
+16 -9
View File
@@ -12,13 +12,14 @@ use std::{
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Vin, Vout, setrlimit,
OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Version, Vin, Vout,
setrlimit,
};
pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_vec::{AnyVec, Compressed, VecIterator};
use brk_parser::Parser;
use brk_vec::{AnyVec, Format, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -33,6 +34,7 @@ pub use vecs::*;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
const COLLISIONS_CHECKED_UP_TO: u32 = 893_000;
const VERSION: Version = Version::ONE;
#[derive(Clone)]
pub struct Indexer {
@@ -40,13 +42,13 @@ pub struct Indexer {
vecs: Option<Vecs>,
stores: Option<Stores>,
check_collisions: bool,
compressed: Compressed,
format: Format,
}
impl Indexer {
pub fn new(
outputs_dir: &Path,
compressed: bool,
format: Format,
check_collisions: bool,
) -> color_eyre::Result<Self> {
setrlimit()?;
@@ -54,7 +56,7 @@ impl Indexer {
path: outputs_dir.to_owned(),
vecs: None,
stores: None,
compressed: Compressed::from(compressed),
format,
check_collisions,
})
}
@@ -62,7 +64,8 @@ impl Indexer {
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::forced_import(
&self.path.join("vecs/indexed"),
self.compressed,
VERSION + Version::ZERO,
self.format,
)?);
Ok(())
}
@@ -70,14 +73,17 @@ impl Indexer {
/// Do NOT import multiple times are things will break !!!
/// Clone struct instead
pub fn import_stores(&mut self) -> color_eyre::Result<()> {
self.stores = Some(Stores::forced_import(&self.path.join("stores"))?);
self.stores = Some(Stores::forced_import(
&self.path.join("stores"),
VERSION + Version::ZERO,
)?);
Ok(())
}
pub fn index(
&mut self,
parser: &Parser,
rpc: &'static rpc::Client,
rpc: &'static bitcoincore_rpc::Client,
exit: &Exit,
) -> color_eyre::Result<Indexes> {
let starting_indexes = Indexes::try_from((
@@ -114,6 +120,7 @@ impl Indexer {
if starting_indexes.height > Height::try_from(rpc)?
|| end.is_some_and(|end| starting_indexes.height > end)
{
info!("Up to date, nothing to index.");
return Ok(starting_indexes);
}
@@ -1,20 +1,15 @@
use std::{fs, path::Path, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex,
TxidPrefix,
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, Result,
TxIndex, TxidPrefix, Value, Version,
};
use brk_vec::{AnyIterableVec, Value, Version};
use brk_store::Store;
use brk_vec::AnyIterableVec;
use fjall::{PersistMode, TransactionalKeyspace};
use crate::Indexes;
mod base;
mod meta;
pub use base::*;
pub use meta::*;
use super::Vecs;
#[derive(Clone)]
@@ -25,41 +20,46 @@ pub struct Stores {
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
}
const VERSION: Version = Version::ZERO;
impl Stores {
pub fn forced_import(path: &Path) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let keyspace = match Self::open_keyspace(path) {
let keyspace = match brk_store::open_keyspace(path) {
Ok(keyspace) => keyspace,
Err(_) => {
fs::remove_dir_all(path)?;
return Self::forced_import(path);
return Self::forced_import(path, version);
}
};
thread::scope(|scope| {
let addressbyteshash_to_outputtypeindex = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"addressbyteshash_to_outputtypeindex",
Version::ZERO,
version + VERSION + Version::ZERO,
None,
)
});
let blockhashprefix_to_height = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"blockhashprefix_to_height",
Version::ZERO,
version + VERSION + Version::ZERO,
None,
)
});
let txidprefix_to_txindex = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"txidprefix_to_txindex",
Version::ZERO,
version + VERSION + Version::ZERO,
None,
)
});
@@ -288,8 +288,8 @@ impl Stores {
.unwrap()
}
pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
thread::scope(|scope| -> fjall::Result<()> {
pub fn commit(&mut self, height: Height) -> Result<()> {
thread::scope(|scope| -> Result<()> {
let addressbyteshash_to_outputtypeindex_commit_handle =
scope.spawn(|| self.addressbyteshash_to_outputtypeindex.commit(height));
let blockhashprefix_to_height_commit_handle =
@@ -306,7 +306,9 @@ impl Stores {
Ok(())
})?;
self.keyspace.persist(PersistMode::SyncAll)
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
}
pub fn rotate_memtables(&self) {
@@ -314,10 +316,4 @@ impl Stores {
self.blockhashprefix_to_height.rotate_memtable();
self.txidprefix_to_txindex.rotate_memtable();
}
fn open_keyspace(path: &Path) -> fjall::Result<TransactionalKeyspace> {
fjall::Config::new(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.open_transactional()
}
}
-207
View File
@@ -1,207 +0,0 @@
use std::{
collections::{BTreeMap, BTreeSet},
error,
fmt::Debug,
mem,
path::Path,
};
use brk_core::Height;
use brk_vec::{Value, Version};
use byteview::ByteView;
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace,
TransactionalPartitionHandle,
};
use zerocopy::{Immutable, IntoBytes};
use super::StoreMeta;
pub struct Store<Key, Value> {
meta: StoreMeta,
name: String,
keyspace: TransactionalKeyspace,
partition: TransactionalPartitionHandle,
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
}
const CHECK_COLLISISONS: bool = true;
const MAJOR_FJALL_VERSION: Version = Version::TWO;
impl<K, V> Store<K, V>
where
K: Debug + Clone + Into<ByteView> + Ord + Immutable + IntoBytes,
V: Debug + Clone + Into<ByteView> + TryFrom<ByteView>,
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
{
pub fn import(
keyspace: TransactionalKeyspace,
path: &Path,
name: &str,
version: Version,
) -> color_eyre::Result<Self> {
let version = MAJOR_FJALL_VERSION + version;
let (meta, partition) = StoreMeta::checked_open(
&keyspace,
&path.join(format!("meta/{name}")),
version,
|| {
Self::open_partition_handle(&keyspace, name).inspect_err(|_| {
eprintln!("Delete {path:?} and try again");
})
},
)?;
let rtx = keyspace.read_tx();
Ok(Self {
meta,
name: name.to_owned(),
keyspace,
partition,
rtx,
puts: BTreeMap::new(),
dels: BTreeSet::new(),
})
}
pub fn get(&self, key: &K) -> color_eyre::Result<Option<Value<V>>> {
if let Some(v) = self.puts.get(key) {
Ok(Some(Value::Ref(v)))
} else if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes())? {
Ok(Some(Value::Owned(V::try_from(slice.as_bytes().into())?)))
} else {
Ok(None)
}
}
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
if !self.dels.is_empty() {
// self.dels.remove(&key);
unreachable!("Shouldn't reach this");
}
self.puts.insert(key, value);
}
}
pub fn remove(&mut self, key: K) {
if self.is_empty() {
return;
}
if !self.puts.is_empty() {
unreachable!("Shouldn't reach this");
}
if !self.dels.insert(key.clone()) {
dbg!(key, &self.meta.path());
unreachable!();
}
}
pub fn commit(&mut self, height: Height) -> Result<()> {
if self.has(height) && self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
}
self.meta.export(self.len(), height)?;
let mut wtx = self.keyspace.write_tx();
mem::take(&mut self.dels)
.into_iter()
.for_each(|key| wtx.remove(&self.partition, key.as_bytes()));
mem::take(&mut self.puts)
.into_iter()
.for_each(|(key, value)| {
if CHECK_COLLISISONS {
#[allow(unused_must_use)]
if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) {
dbg!(
&key,
V::try_from(value.as_bytes().into()).unwrap(),
&self.meta,
self.rtx.get(&self.partition, key.as_bytes())
);
unreachable!();
}
}
wtx.insert(
&self.partition,
key.as_bytes(),
&*ByteView::try_from(value).unwrap(),
)
});
wtx.commit()?;
self.rtx = self.keyspace.read_tx();
Ok(())
}
pub fn rotate_memtable(&self) {
let _ = self.partition.inner().rotate_memtable();
}
pub fn height(&self) -> Option<Height> {
self.meta.height()
}
pub fn len(&self) -> usize {
self.meta.len() + self.puts.len() - self.dels.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn has(&self, height: Height) -> bool {
self.meta.has(height)
}
pub fn needs(&self, height: Height) -> bool {
self.meta.needs(height)
}
fn open_partition_handle(
keyspace: &TransactionalKeyspace,
name: &str,
) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(
name,
PartitionCreateOptions::default()
.bloom_filter_bits(Some(5))
.max_memtable_size(8 * 1024 * 1024)
.manual_journal_persist(true),
)
}
pub fn reset_partition(&mut self) -> Result<()> {
self.keyspace.delete_partition(self.partition.clone())?;
self.keyspace.persist(PersistMode::SyncAll)?;
self.partition = Self::open_partition_handle(&self.keyspace, &self.name)?;
Ok(())
}
}
impl<Key, Value> Clone for Store<Key, Value>
where
Key: Clone,
Value: Clone,
{
fn clone(&self) -> Self {
Self {
meta: self.meta.clone(),
name: self.name.clone(),
keyspace: self.keyspace.clone(),
partition: self.partition.clone(),
rtx: self.keyspace.read_tx(),
puts: self.puts.clone(),
dels: self.dels.clone(),
}
}
}
@@ -4,14 +4,17 @@ use brk_core::{
AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, OutputTypeIndex, P2ABytes, P2AIndex, P2MSIndex, P2PK33Bytes, P2PK33Index,
P2PK65Bytes, P2PK65Index, P2PKHBytes, P2PKHIndex, P2SHBytes, P2SHIndex, P2TRBytes, P2TRIndex,
P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Sats, StoredF64, StoredU32,
StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Weight,
P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Result, Sats, StoredF64,
StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Version,
Weight,
};
use brk_vec::{AnyCollectableVec, AnyIndexedVec, Compressed, IndexedVec, Result, Version};
use brk_vec::{AnyCollectableVec, AnyIndexedVec, Format, IndexedVec};
use rayon::prelude::*;
use crate::Indexes;
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub emptyoutputindex_to_txindex: IndexedVec<EmptyOutputIndex, TxIndex>,
@@ -63,273 +66,282 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
version: Version,
format: Format,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
emptyoutputindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_blockhash: IndexedVec::forced_import(
path,
"blockhash",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_difficulty: IndexedVec::forced_import(
path,
"difficulty",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_emptyoutputindex: IndexedVec::forced_import(
path,
"first_emptyoutputindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_inputindex: IndexedVec::forced_import(
path,
"first_inputindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_opreturnindex: IndexedVec::forced_import(
path,
"first_opreturnindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_outputindex: IndexedVec::forced_import(
path,
"first_outputindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2aindex: IndexedVec::forced_import(
path,
"first_p2aindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2msindex: IndexedVec::forced_import(
path,
"first_p2msindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2pk33index: IndexedVec::forced_import(
path,
"first_p2pk33index",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2pk65index: IndexedVec::forced_import(
path,
"first_p2pk65index",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2pkhindex: IndexedVec::forced_import(
path,
"first_p2pkhindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2shindex: IndexedVec::forced_import(
path,
"first_p2shindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2trindex: IndexedVec::forced_import(
path,
"first_p2trindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2wpkhindex: IndexedVec::forced_import(
path,
"first_p2wpkhindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_p2wshindex: IndexedVec::forced_import(
path,
"first_p2wshindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_txindex: IndexedVec::forced_import(
path,
"first_txindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_first_unknownoutputindex: IndexedVec::forced_import(
path,
"first_unknownoutputindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_timestamp: IndexedVec::forced_import(
path,
"timestamp",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_total_size: IndexedVec::forced_import(
path,
"total_size",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
height_to_weight: IndexedVec::forced_import(
path,
"weight",
version + VERSION + Version::ZERO,
format,
)?,
height_to_weight: IndexedVec::forced_import(path, "weight", Version::ZERO, compressed)?,
inputindex_to_outputindex: IndexedVec::forced_import(
path,
"outputindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
opreturnindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_outputtype: IndexedVec::forced_import(
path,
"outputtype",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_outputtypeindex: IndexedVec::forced_import(
path,
"outputtypeindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_value: IndexedVec::forced_import(
path,
"value",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
p2aindex_to_p2abytes: IndexedVec::forced_import(
path,
"p2abytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2msindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
path,
"p2pk33bytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk65index_to_p2pk65bytes: IndexedVec::forced_import(
path,
"p2pk65bytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pkhindex_to_p2pkhbytes: IndexedVec::forced_import(
path,
"p2pkhbytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2shindex_to_p2shbytes: IndexedVec::forced_import(
path,
"p2shbytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2trindex_to_p2trbytes: IndexedVec::forced_import(
path,
"p2trbytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wpkhindex_to_p2wpkhbytes: IndexedVec::forced_import(
path,
"p2wpkhbytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wshindex_to_p2wshbytes: IndexedVec::forced_import(
path,
"p2wshbytes",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_base_size: IndexedVec::forced_import(
path,
"base_size",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_first_inputindex: IndexedVec::forced_import(
path,
"first_inputindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_first_outputindex: IndexedVec::forced_import(
path,
"first_outputindex",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_is_explicitly_rbf: IndexedVec::forced_import(
path,
"is_explicitly_rbf",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_rawlocktime: IndexedVec::forced_import(
path,
"rawlocktime",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_total_size: IndexedVec::forced_import(
path,
"total_size",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_txid: IndexedVec::forced_import(
path,
"txid",
Version::ZERO,
Compressed::NO,
version + VERSION + Version::ZERO,
Format::Raw,
)?,
txindex_to_txversion: IndexedVec::forced_import(
path,
"txversion",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
unknownoutputindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
Version::ZERO,
compressed,
version + VERSION + Version::ZERO,
format,
)?,
})
}
pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> {
pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> Result<()> {
let saved_height = starting_indexes.height.decremented().unwrap_or_default();
let &Indexes {
@@ -447,7 +459,7 @@ impl Vecs {
&mut self,
index: OutputTypeIndex,
bytes: AddressBytes,
) -> brk_vec::Result<()> {
) -> Result<()> {
match bytes {
AddressBytes::P2PK65(bytes) => self
.p2pk65index_to_p2pk65bytes
+1
View File
@@ -4,6 +4,7 @@ description = "A clean logger used in the Bitcoin Research Kit"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
+1
View File
@@ -6,6 +6,7 @@ categories = ["cryptography::cryptocurrencies", "encoding"]
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
-3
View File
@@ -13,9 +13,6 @@ use brk_core::Height;
use crossbeam::channel::{Receiver, bounded};
use rayon::prelude::*;
pub use bitcoin;
pub use bitcoincore_rpc as rpc;
mod blk_index_to_blk_path;
mod blk_index_to_blk_recap;
mod blk_metadata;
+2
View File
@@ -4,9 +4,11 @@ description = "A library that finds requested datasets"
license.workspace = true
edition.workspace = true
version.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_computer = { workspace = true }
brk_indexer = { workspace = true }
brk_vec = { workspace = true }
-3
View File
@@ -4,9 +4,6 @@
<a href="https://github.com/bitcoinresearchkit/brk">
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
</a>
<a href="https://kibo.money">
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
</a>
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
<img src="https://img.shields.io/crates/l/brk" alt="License" />
</a>
+4 -4
View File
@@ -3,19 +3,19 @@ use std::path::Path;
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_query::{Index, Query};
use brk_vec::Computation;
use brk_vec::{Computation, Format};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let outputs_dir = Path::new("../../_outputs");
let compressed = true;
let format = Format::Compressed;
let mut indexer = Indexer::new(outputs_dir, compressed, true)?;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir, None, compressed);
let mut computer = Computer::new(outputs_dir, None, format);
computer.import_vecs(&indexer, Computation::Lazy)?;
let query = Query::build(&indexer, &computer);
+3 -2
View File
@@ -4,6 +4,7 @@
#![doc = "```"]
use brk_computer::Computer;
use brk_core::Result;
use brk_indexer::Indexer;
use brk_vec::AnyCollectableVec;
use tabled::settings::Style;
@@ -92,10 +93,10 @@ impl<'a> Query<'a> {
) -> color_eyre::Result<Output> {
let mut values = vecs
.iter()
.map(|(_, vec)| -> brk_vec::Result<Vec<serde_json::Value>> {
.map(|(_, vec)| -> Result<Vec<serde_json::Value>> {
vec.collect_range_serde_json(from, to)
})
.collect::<brk_vec::Result<Vec<_>>>()?;
.collect::<Result<Vec<_>>>()?;
if values.is_empty() {
return Ok(Output::default(format));
+14 -2
View File
@@ -18,8 +18,20 @@ impl<'a> VecTrees<'a> {
let split = name.split("_to_").collect::<Vec<_>>();
if split.len() != 2
&& !(split.len() == 3
&& (split.get(1) == Some(&"up")
|| split.get(1).is_some_and(|s| s.starts_with("from"))))
&& split.get(1).is_some_and(|s| {
s == &"up"
|| s == &"start"
|| s.ends_with("relative")
|| s.starts_with("from")
|| s == &"cumulative_up"
|| s.starts_with("cumulative_start")
|| s.starts_with("cumulative_from")
}))
&& !(split.len() == 4
&& split
.get(1)
.is_some_and(|s| s == &"up" || s == &"start" || s.starts_with("from"))
&& split.get(2).is_some_and(|s| s.ends_with("relative")))
{
dbg!(&name, &split);
panic!();

Some files were not shown because too many files have changed in this diff Show More