diff --git a/.gitignore b/.gitignore
index af98ef201..d0bd3df09 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,44 +1,19 @@
# Mac OS
.DS_Store
-# To do
-/charts
-TODO.md
-
# Builds
-dist
target
-# I/O
-in
-out
-.log
-/datasets
-/datasets2
-/price
-*..*
-/txout_*
-/db
-
-# Sync
-.stfolder
-
# Copies
*\ copy*
# Ignored
-ignore
-
-# Scripts
-/start-node.sh
+/_*
# Editors
.vscode
.zed
-# Configs
-config.toml
-
# Flamegraph
flamegraph/
flamegraph.svg
@@ -53,7 +28,11 @@ snapshots*/
docker/kibo
# Types
-website/scripts/types/paths.d.ts
+paths.d.ts
+vecid-to-indexes.d.ts
-# Misc
-OPENSATS.md
+# Outputs
+_outputs
+
+# Python
+.ropeproject
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f703c65c..1714fa916 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,13 +1,44 @@
-# Changelog
-
-## v. 0.5.0 | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
+# v0.6.0 | WIP | A new beginning
-
+## Global
+
+- Completely redesign the back-end
+
+- Merged parser and server crates into a single project (and thus executable), so now both will run at the same time with a single `cargo run -r` [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
+- Added `--no-server` and `--no-parser` to disable each if needed
+- Improved executable parameters
+- Started using `log` and `env_logger` crates instead of custom code [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
+- Improved logs
+- Fixed input being unfocused right after being focused in Brave browser [#9a9ae61](https://github.com/kibo-money/kibo/commit/9a9ae614d07b54c08b7e9c0e2aefe3b52fdb93c5)
+
+- Reworked server's API code [#6ab0f46]( https://github.com/kibo-money/kibo/commit/6ab0f463119a902a1b7ca9691b54f61543bb8f2f)
+ - New route format: `/api/date-to-realized-price` is now `/api/realized-price?kind=date`
+ - Added status and timing to logs
+- Updated website packages
+- Added API support for datasets by timestamp (by merging any dataset by height with the height to timestamp dataset and so it still uses heights as chunk ids) [#ca00f3f](https://github.com/kibo-money/kibo/commit/ca00f3f71526f0c5c16021024fec7e5c6e47221c)
+ - `/api/realized-price?kind=t`
+ - `/api/realized-price?kind=timestamp&chunk=860000`
+- Created separate crate for indexing called `bindex`
+- Created a crate a storage engine specialized in storing datasets that have indexes as keys and thus can be represented by an array/vec called `storable-vec`
+- Removed the need for the `-txindex=1` parameter when starting your Bitcoin Core node as kibō has its own indexes now
+
+## Git
+
+Added git tags for each version though Markdown won't display formatted on Github so left the default text
+
+## Deprecated
+
+Moved Sanakirja database wrapper to its own crate (`snkrj`) and added a robust auto defragmentation to improve disk usage without the need for user's intervention.
+Since it's not used anymore it will moved out of the repository relatively soon.
+
+# [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
+
+
## Datasets
@@ -72,15 +103,15 @@
- Moved back to this repo
-## v. 0.4.0 | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
+# [v0.4.0](https://github.com/kibo-money/kibo/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
-
+
-### Brand
+## Brand
- **Satonomics** is now **kibō** 🎉
-### Website
+## Website
- Complete redesign of the website
- Rewrote the whole application and removed `node`/`npm`/`pnpm` dependencies in favor for pure `HTML`/`CSS`/`Javascript`
@@ -88,7 +119,7 @@
- Added Trading View attribution link to the settings frame and file in the lightweight charts folder
- Many other changes
-### Parser
+## Parser
- Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
- Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage
@@ -103,17 +134,17 @@
- Various first run fixes
- Added to `-h` which arguments are saved, which is all of them at the time of writing
-### Server
+## Server
- Updated the code to support compressed binaries
- Added serving of the website
- Improved `Cache-Control` behavior
-## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
+# [v0.3.0](https://github.com/kibo-money/kibo/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
-
+
-### Parser
+## Parser
- Global
- Improved self-hosting by:
@@ -156,7 +187,7 @@
- Price
- Improved error message when price cannot be found
-### App
+## App
- General
- Added chart scroll button for nice animations à la Wicked
@@ -182,17 +213,17 @@
- Settings
- Removed the horizontal scroll bar which was unintended
-### Server
+## Server
- Run file
- Only run with a watcher if `cargo watch` is available
- Removed id_to_path file in favor for only `paths.d.ts` in `app/src/types`
-## v. 0.2.0 | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
+# [v0.2.0](https://github.com/kibo-money/kibo/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
-
+
-### App
+## App
- General
- Added the height version of all datasets and many optimizations to make them usable but only available on desktop and tablets for now
@@ -220,24 +251,24 @@
- Hopefully made scrollbars a little more subtle on WIndows and Linux, can't test
- Generale style updates
-### Parser
+## Parser
- Fixed ulimit only being run in Mac OS instead of whenever the program is detected
-## v. 0.1.1 | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
+# [v0.1.1](https://github.com/kibo-money/kibo/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
-
+
-### Parser
+## Parser
- Fixed overflow in `Price` struct which caused many Realized Caps and Realized Prices to have completely bogus data
- Fixed Realized Cap computation which was using rounded prices instead normal ones
-### Server
+## Server
- Added the chunk, date and time of the request to the terminal logs
-### App
+## App
- Chart
- Added double click option on a legend to toggle the visibility of all other series
@@ -270,14 +301,14 @@
- Misc
- Removed tracker even though it was a very privacy friendly as it appeared to not be working properly
-### Price
+## Price
- Deleted old price datasets and their backups
-## v. 0.1.0 | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
+# [v0.1.0](https://github.com/kibo-money/kibo/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
-
+
-## v. 0.0.X | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
+# v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
-
+
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index a6c0043c3..000000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Guidelines
-
-## Parser
-
-- Avoid floats as much as possible
- - Use structs like `WAmount` and `Price` for calculations
- - **Only** use `WAmount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
-- No `Arc`, `Rc`, `Mutex` even from third party libraries, they're slower
diff --git a/parser/Cargo.lock b/Cargo.lock
similarity index 60%
rename from parser/Cargo.lock
rename to Cargo.lock
index 91514253c..e6fd1bc01 100644
--- a/parser/Cargo.lock
+++ b/Cargo.lock
@@ -1,6 +1,6 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
-version = 3
+version = 4
[[package]]
name = "addr2line"
@@ -18,59 +18,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
-name = "ahash"
-version = "0.8.11"
+name = "adler2"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
- "cfg-if",
- "getrandom",
- "once_cell",
- "version_check",
- "zerocopy",
+ "memchr",
]
[[package]]
-name = "allocative"
-version = "0.3.3"
+name = "alloc-no-stdlib"
+version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "082af274fd02beef17b7f0725a49ecafe6c075ef56cac9d6363eb3916a9817ae"
+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
- "allocative_derive",
- "ctor",
+ "alloc-no-stdlib",
]
[[package]]
-name = "allocative_derive"
-version = "0.3.3"
+name = "allocator-api2"
+version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.85",
-]
-
-[[package]]
-name = "android-tzdata"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
-
-[[package]]
-name = "android_system_properties"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
-dependencies = [
- "libc",
-]
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "anstream"
-version = "0.6.15"
+version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -83,36 +70,37 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.8"
+version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
[[package]]
name = "anstyle-parse"
-version = "0.2.5"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
-version = "1.1.1"
+version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
name = "anstyle-wincon"
-version = "3.0.4"
+version = "3.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
dependencies = [
"anstyle",
- "windows-sys 0.52.0",
+ "once_cell",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -122,10 +110,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
-name = "atomic-waker"
-version = "1.1.2"
+name = "assert-unchecked"
+version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+checksum = "7330592adf847ee2e3513587b4db2db410a0d751378654e7e993d9adcbe5c795"
+
+[[package]]
+name = "async-compression"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522"
+dependencies = [
+ "brotli",
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+ "zstd",
+ "zstd-safe",
+]
[[package]]
name = "autocfg"
@@ -133,6 +137,60 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
+[[package]]
+name = "axum"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8"
+dependencies = [
+ "axum-core",
+ "bytes",
+ "form_urlencoded",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "serde_json",
+ "serde_path_to_error",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "tokio",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "rustversion",
+ "sync_wrapper",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
[[package]]
name = "backtrace"
version = "0.3.71"
@@ -143,7 +201,7 @@ dependencies = [
"cc",
"cfg-if",
"libc",
- "miniz_oxide",
+ "miniz_oxide 0.7.4",
"object",
"rustc-demangle",
]
@@ -165,10 +223,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
-name = "base64"
-version = "0.22.1"
+name = "base64-simd"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
+dependencies = [
+ "outref",
+ "vsimd",
+]
[[package]]
name = "bech32"
@@ -176,29 +238,11 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
-[[package]]
-name = "bincode"
-version = "2.0.0-rc.3"
-source = "git+https://github.com/bincode-org/bincode.git#5a91c1210168b968b957a14b743cf134f2d20719"
-dependencies = [
- "bincode_derive",
- "serde",
- "unty",
-]
-
-[[package]]
-name = "bincode_derive"
-version = "2.0.0-rc.3"
-source = "git+https://github.com/bincode-org/bincode.git#5a91c1210168b968b957a14b743cf134f2d20719"
-dependencies = [
- "virtue",
-]
-
[[package]]
name = "bitcoin"
-version = "0.32.3"
+version = "0.32.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0032b0e8ead7074cda7fc4f034409607e3f03a6f71d66ade8a307f79b4d99e73"
+checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026"
dependencies = [
"base58ck",
"bech32",
@@ -223,9 +267,9 @@ dependencies = [
[[package]]
name = "bitcoin-io"
-version = "0.1.2"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "340e09e8399c7bd8912f495af6aa58bea0c9214773417ffaa8f6460f93aaee56"
+checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf"
[[package]]
name = "bitcoin-units"
@@ -273,41 +317,156 @@ dependencies = [
]
[[package]]
-name = "biter"
-version = "0.1.1"
+name = "bitflags"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
+
+[[package]]
+name = "brk"
+version = "0.0.0"
dependencies = [
- "bitcoin",
- "bitcoincore-rpc",
- "crossbeam",
- "derived-deref",
- "rayon",
- "serde",
- "serde_json",
+ "brk_computer",
+ "brk_fetcher",
+ "brk_indexer",
+ "brk_logger",
+ "brk_parser",
+ "brk_server",
]
[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+name = "brk_cli"
+version = "0.0.0"
[[package]]
-name = "bitflags"
-version = "2.6.0"
+name = "brk_computer"
+version = "0.0.0"
+dependencies = [
+ "brk_fetcher",
+ "brk_indexer",
+ "brk_parser",
+ "byteview",
+ "color-eyre",
+ "derive_deref",
+ "fjall",
+ "hodor",
+ "storable_vec",
+ "zerocopy 0.8.20",
+]
+
+[[package]]
+name = "brk_fetcher"
+version = "0.0.0"
+dependencies = [
+ "brk_indexer",
+ "brk_logger",
+ "color-eyre",
+ "derive_deref",
+ "jiff",
+ "log",
+ "minreq",
+ "serde",
+ "serde_json",
+ "storable_vec",
+ "zerocopy 0.8.20",
+]
+
+[[package]]
+name = "brk_indexer"
+version = "0.0.0"
+dependencies = [
+ "bitcoin",
+ "brk_logger",
+ "brk_parser",
+ "byteview",
+ "color-eyre",
+ "derive_deref",
+ "fjall",
+ "hodor",
+ "jiff",
+ "log",
+ "rapidhash",
+ "rayon",
+ "rlimit",
+ "serde",
+ "serde_bytes",
+ "storable_vec",
+ "zerocopy 0.8.20",
+]
+
+[[package]]
+name = "brk_logger"
+version = "0.0.0"
+dependencies = [
+ "color-eyre",
+ "env_logger",
+ "jiff",
+]
+
+[[package]]
+name = "brk_parser"
+version = "0.0.0"
+dependencies = [
+ "bitcoin",
+ "bitcoincore-rpc",
+ "byteview",
+ "crossbeam",
+ "derive_deref",
+ "rayon",
+ "serde",
+ "serde_json",
+ "zerocopy 0.8.20",
+]
+
+[[package]]
+name = "brk_server"
+version = "0.0.0"
+dependencies = [
+ "axum",
+ "brk_computer",
+ "brk_indexer",
+ "brk_logger",
+ "color-eyre",
+ "derive_deref",
+ "jiff",
+ "log",
+ "oxc",
+ "serde",
+ "serde_json",
+ "storable_vec",
+ "tokio",
+ "tower-http",
+]
+
+[[package]]
+name = "brotli"
+version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "4.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
[[package]]
name = "bumpalo"
-version = "3.16.0"
+version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
-
-[[package]]
-name = "bytemuck"
-version = "1.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae"
+checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
+dependencies = [
+ "allocator-api2",
+]
[[package]]
name = "byteorder"
@@ -317,15 +476,30 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
-version = "1.7.2"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
+checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
+
+[[package]]
+name = "byteview"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a4516a8561bff0598c45512f90ee04ed62cee2cb36839e650a0a0704d5f741f"
+
+[[package]]
+name = "castaway"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
+dependencies = [
+ "rustversion",
+]
[[package]]
name = "cc"
-version = "1.1.28"
+version = "1.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1"
+checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af"
dependencies = [
"jobserver",
"libc",
@@ -344,61 +518,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
-[[package]]
-name = "chrono"
-version = "0.4.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
-dependencies = [
- "android-tzdata",
- "iana-time-zone",
- "js-sys",
- "num-traits",
- "serde",
- "wasm-bindgen",
- "windows-targets",
-]
-
-[[package]]
-name = "clap"
-version = "4.5.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
-dependencies = [
- "clap_builder",
- "clap_derive",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.5.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim",
-]
-
-[[package]]
-name = "clap_derive"
-version = "4.5.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "syn 2.0.85",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
-
[[package]]
name = "color-eyre"
version = "0.6.3"
@@ -410,7 +529,7 @@ dependencies = [
"eyre",
"indenter",
"once_cell",
- "owo-colors",
+ "owo-colors 3.5.0",
"tracing-error",
]
@@ -421,32 +540,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2"
dependencies = [
"once_cell",
- "owo-colors",
+ "owo-colors 3.5.0",
"tracing-core",
"tracing-error",
]
[[package]]
name = "colorchoice"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
-name = "core-foundation"
-version = "0.9.4"
+name = "compact_str"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32"
dependencies = [
- "core-foundation-sys",
- "libc",
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "static_assertions",
]
[[package]]
-name = "core-foundation-sys"
-version = "0.8.7"
+name = "compare"
+version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
+
+[[package]]
+name = "cow-utils"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "417bef24afe1460300965a25ff4a24b8b45ad011948302ec221e8a0a81eb2c79"
+
+[[package]]
+name = "crc32fast"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+dependencies = [
+ "cfg-if",
+]
[[package]]
name = "crossbeam"
@@ -463,18 +601,18 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
-version = "0.5.13"
+version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
-version = "0.8.5"
+version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
@@ -491,28 +629,28 @@ dependencies = [
[[package]]
name = "crossbeam-queue"
-version = "0.3.11"
+version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
+checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"crossbeam-utils",
]
[[package]]
-name = "crossbeam-utils"
-version = "0.8.20"
+name = "crossbeam-skiplist"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
[[package]]
-name = "ctor"
-version = "0.1.26"
+name = "crossbeam-utils"
+version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
-dependencies = [
- "quote",
- "syn 1.0.109",
-]
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "ctrlc"
@@ -535,7 +673,7 @@ dependencies = [
"hashbrown 0.14.5",
"lock_api",
"once_cell",
- "parking_lot_core 0.9.10",
+ "parking_lot_core",
]
[[package]]
@@ -550,15 +688,10 @@ dependencies = [
]
[[package]]
-name = "derived-deref"
-version = "2.1.0"
+name = "double-ended-peekable"
+version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.85",
-]
+checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]]
name = "either"
@@ -567,47 +700,54 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
[[package]]
-name = "encoding_rs"
-version = "0.8.34"
+name = "enum_dispatch"
+version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
+checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
dependencies = [
- "cfg-if",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
]
[[package]]
name = "env_filter"
-version = "0.1.2"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
+checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
dependencies = [
"log",
+ "regex",
]
[[package]]
name = "env_logger"
-version = "0.11.5"
+version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
+checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0"
dependencies = [
+ "anstream",
+ "anstyle",
"env_filter",
+ "humantime",
"log",
]
[[package]]
name = "equivalent"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
-version = "0.3.9"
+version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
dependencies = [
"libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -622,9 +762,42 @@ dependencies = [
[[package]]
name = "fastrand"
-version = "2.1.1"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
+[[package]]
+name = "fixedbitset"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
+
+[[package]]
+name = "fjall"
+version = "2.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33e0fd3f3a8cbaa2b179ccc690ea0d37282d24787c06eab0dfd9137e1c4d4699"
+dependencies = [
+ "byteorder",
+ "byteview",
+ "dashmap",
+ "log",
+ "lsm-tree",
+ "path-absolutize",
+ "std-semaphore",
+ "tempfile",
+ "xxhash-rust",
+]
+
+[[package]]
+name = "flate2"
+version = "1.0.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide 0.8.5",
+]
[[package]]
name = "fnv"
@@ -632,21 +805,6 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@@ -656,16 +814,6 @@ dependencies = [
"percent-encoding",
]
-[[package]]
-name = "fs2"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
-dependencies = [
- "libc",
- "winapi",
-]
-
[[package]]
name = "futures-channel"
version = "0.3.31"
@@ -673,7 +821,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
- "futures-sink",
]
[[package]]
@@ -682,12 +829,6 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
-[[package]]
-name = "futures-io"
-version = "0.3.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
-
[[package]]
name = "futures-sink"
version = "0.3.31"
@@ -707,13 +848,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
"futures-core",
- "futures-io",
- "futures-sink",
"futures-task",
- "memchr",
"pin-project-lite",
"pin-utils",
- "slab",
]
[[package]]
@@ -724,7 +861,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.13.3+wasi-0.2.2",
+ "windows-targets",
]
[[package]]
@@ -734,23 +883,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
[[package]]
-name = "h2"
-version = "0.4.6"
+name = "guardian"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
-dependencies = [
- "atomic-waker",
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "http",
- "indexmap",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
-]
+checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
[[package]]
name = "hashbrown"
@@ -760,27 +896,12 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
-version = "0.15.0"
+version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
-
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
-[[package]]
-name = "hermit-abi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
-
-[[package]]
-name = "hermit-abi"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+dependencies = [
+ "allocator-api2",
+]
[[package]]
name = "hex-conservative"
@@ -797,11 +918,19 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
+[[package]]
+name = "hodor"
+version = "0.1.0"
+dependencies = [
+ "ctrlc",
+ "log",
+]
+
[[package]]
name = "http"
-version = "1.1.0"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
+checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea"
dependencies = [
"bytes",
"fnv",
@@ -833,113 +962,55 @@ dependencies = [
[[package]]
name = "httparse"
-version = "1.9.5"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946"
+checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a"
+
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
-version = "1.4.1"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
+checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
- "h2",
"http",
"http-body",
"httparse",
+ "httpdate",
"itoa",
"pin-project-lite",
"smallvec",
"tokio",
- "want",
-]
-
-[[package]]
-name = "hyper-rustls"
-version = "0.27.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
-dependencies = [
- "futures-util",
- "http",
- "hyper",
- "hyper-util",
- "rustls",
- "rustls-pki-types",
- "tokio",
- "tokio-rustls",
- "tower-service",
-]
-
-[[package]]
-name = "hyper-tls"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
-dependencies = [
- "bytes",
- "http-body-util",
- "hyper",
- "hyper-util",
- "native-tls",
- "tokio",
- "tokio-native-tls",
- "tower-service",
]
[[package]]
name = "hyper-util"
-version = "0.1.9"
+version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b"
+checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
dependencies = [
"bytes",
- "futures-channel",
"futures-util",
"http",
"http-body",
"hyper",
"pin-project-lite",
- "socket2",
"tokio",
"tower-service",
- "tracing",
-]
-
-[[package]]
-name = "iana-time-zone"
-version = "0.1.61"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
-dependencies = [
- "android_system_properties",
- "core-foundation-sys",
- "iana-time-zone-haiku",
- "js-sys",
- "wasm-bindgen",
- "windows-core",
-]
-
-[[package]]
-name = "iana-time-zone-haiku"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "idna"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
]
[[package]]
@@ -950,61 +1021,21 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]]
name = "indexmap"
-version = "2.6.0"
+version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
+checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
dependencies = [
"equivalent",
- "hashbrown 0.15.0",
+ "hashbrown 0.15.2",
]
[[package]]
-name = "inferno"
-version = "0.11.21"
+name = "interval-heap"
+version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
+checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
dependencies = [
- "ahash",
- "clap",
- "crossbeam-channel",
- "crossbeam-utils",
- "dashmap",
- "env_logger",
- "indexmap",
- "is-terminal",
- "itoa",
- "log",
- "num-format",
- "once_cell",
- "quick-xml",
- "rgb",
- "str_stack",
-]
-
-[[package]]
-name = "instant"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "ipnet"
-version = "2.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708"
-
-[[package]]
-name = "is-terminal"
-version = "0.4.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b"
-dependencies = [
- "hermit-abi 0.4.0",
- "libc",
- "windows-sys 0.52.0",
+ "compare",
]
[[package]]
@@ -1015,18 +1046,47 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]]
name = "itertools"
-version = "0.13.0"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
dependencies = [
"either",
]
[[package]]
name = "itoa"
-version = "1.0.11"
+version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
+
+[[package]]
+name = "jiff"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3590fea8e9e22d449600c9bbd481a8163bef223e4ff938e5f55899f8cf1adb93"
+dependencies = [
+ "jiff-tzdb-platform",
+ "log",
+ "portable-atomic",
+ "portable-atomic-util",
+ "serde",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "jiff-tzdb"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf2cec2f5d266af45a071ece48b1fb89f3b00b2421ac3a5fe10285a6caaa60d3"
+
+[[package]]
+name = "jiff-tzdb-platform"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a63c62e404e7b92979d2792352d885a7f8f83fd1d0d31eea582d77b2ceca697e"
+dependencies = [
+ "jiff-tzdb",
+]
[[package]]
name = "jobserver"
@@ -1037,22 +1097,13 @@ dependencies = [
"libc",
]
-[[package]]
-name = "js-sys"
-version = "0.3.70"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
-dependencies = [
- "wasm-bindgen",
-]
-
[[package]]
name = "jsonrpc"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
dependencies = [
- "base64 0.13.1",
+ "base64",
"minreq",
"serde",
"serde_json",
@@ -1066,15 +1117,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
-version = "0.2.159"
+version = "0.2.170"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
+checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828"
[[package]]
name = "linux-raw-sys"
-version = "0.4.14"
+version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "lock_api"
@@ -1088,9 +1139,45 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.22"
+version = "0.4.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
+checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
+
+[[package]]
+name = "lsm-tree"
+version = "2.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "614a43954a8414dcca688f9d4c91f727860b55ffd9a029438b18f234ed33330f"
+dependencies = [
+ "byteorder",
+ "crossbeam-skiplist",
+ "double-ended-peekable",
+ "enum_dispatch",
+ "guardian",
+ "interval-heap",
+ "log",
+ "lz4_flex",
+ "path-absolutize",
+ "quick_cache",
+ "rustc-hash",
+ "self_cell",
+ "tempfile",
+ "value-log",
+ "varint-rs",
+ "xxhash-rust",
+]
+
+[[package]]
+name = "lz4_flex"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
+
+[[package]]
+name = "matchit"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "memchr"
@@ -1123,65 +1210,75 @@ dependencies = [
]
[[package]]
-name = "minreq"
-version = "2.12.0"
+name = "miniz_oxide"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0"
+checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
+dependencies = [
+ "adler2",
+]
+
+[[package]]
+name = "minreq"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da0c420feb01b9fb5061f8c8f452534361dd783756dcf38ec45191ce55e7a161"
dependencies = [
"log",
+ "once_cell",
+ "rustls",
+ "rustls-webpki",
"serde",
"serde_json",
+ "webpki-roots",
]
[[package]]
name = "mio"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
+checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
dependencies = [
- "hermit-abi 0.3.9",
"libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.52.0",
]
-[[package]]
-name = "native-tls"
-version = "0.2.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466"
-dependencies = [
- "libc",
- "log",
- "openssl",
- "openssl-probe",
- "openssl-sys",
- "schannel",
- "security-framework",
- "security-framework-sys",
- "tempfile",
-]
-
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
-name = "num-format"
-version = "0.4.4"
+name = "nonmax"
+version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3"
+checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51"
+
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
- "arrayvec",
- "itoa",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
]
[[package]]
@@ -1204,62 +1301,15 @@ dependencies = [
[[package]]
name = "once_cell"
-version = "1.20.2"
+version = "1.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e"
[[package]]
-name = "openssl"
-version = "0.10.66"
+name = "outref"
+version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
-dependencies = [
- "bitflags 2.6.0",
- "cfg-if",
- "foreign-types",
- "libc",
- "once_cell",
- "openssl-macros",
- "openssl-sys",
-]
-
-[[package]]
-name = "openssl-macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.85",
-]
-
-[[package]]
-name = "openssl-probe"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
-
-[[package]]
-name = "openssl-sys"
-version = "0.9.103"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
-[[package]]
-name = "ordered-float"
-version = "4.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83e7ccb95e240b7c9506a3d544f10d935e142cc90b0a1d56954fb44d89ad6b97"
-dependencies = [
- "num-traits",
-]
+checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "owo-colors"
@@ -1268,28 +1318,353 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
-name = "parking_lot"
-version = "0.11.2"
+name = "owo-colors"
+version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
+checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
+
+[[package]]
+name = "oxc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6be05b94a99c886e3c8f79c0330e746df6fe27f8440b498ea7a5eb2fee2f67"
dependencies = [
- "instant",
- "lock_api",
- "parking_lot_core 0.8.6",
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_codegen",
+ "oxc_diagnostics",
+ "oxc_mangler",
+ "oxc_minifier",
+ "oxc_parser",
+ "oxc_regular_expression",
+ "oxc_span",
+ "oxc_syntax",
]
[[package]]
-name = "parking_lot_core"
-version = "0.8.6"
+name = "oxc-miette"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
+checksum = "e03e63fd113c068b82d07c9c614b0b146c08a3ac0a4dface3ea1d1a9d14d549e"
dependencies = [
"cfg-if",
- "instant",
- "libc",
- "redox_syscall 0.2.16",
- "smallvec",
- "winapi",
+ "owo-colors 4.2.0",
+ "oxc-miette-derive",
+ "textwrap",
+ "thiserror",
+ "unicode-width 0.2.0",
+]
+
+[[package]]
+name = "oxc-miette-derive"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e21f680e8c5f1900297d394627d495351b9e37761f7bbf90116bd5eeb6e80967"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
+
+[[package]]
+name = "oxc_allocator"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf460405a383f3f7ac134a0e06afdb3525c8fa8f119453167f179ea6b12faaf6"
+dependencies = [
+ "allocator-api2",
+ "bumpalo",
+ "hashbrown 0.15.2",
+ "rustc-hash",
+ "simdutf8",
+]
+
+[[package]]
+name = "oxc_ast"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b273094a3e96e84d3d8ce82190fd7113f70bb2a8dcb652c78b46df79fdd156cc"
+dependencies = [
+ "bitflags",
+ "cow-utils",
+ "nonmax",
+ "oxc_allocator",
+ "oxc_ast_macros",
+ "oxc_estree",
+ "oxc_regular_expression",
+ "oxc_span",
+ "oxc_syntax",
+]
+
+[[package]]
+name = "oxc_ast_macros"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab1dff20655433e64452ee12a9c87dba5d307a47fc8e155b4e37d746d66b37dd"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
+
+[[package]]
+name = "oxc_cfg"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f289d7b40c97d1b7d2a8cf1528cd5af0f3f4e832dffdec20b6485b9d6240de0b"
+dependencies = [
+ "bitflags",
+ "itertools",
+ "nonmax",
+ "oxc_index",
+ "oxc_syntax",
+ "petgraph",
+ "rustc-hash",
+]
+
+[[package]]
+name = "oxc_codegen"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8aa20c10288eacf1a8ec93981198c1428c7c9d0436ae7e0f0037a02e87535c94"
+dependencies = [
+ "assert-unchecked",
+ "bitflags",
+ "cow-utils",
+ "nonmax",
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_data_structures",
+ "oxc_index",
+ "oxc_semantic",
+ "oxc_sourcemap",
+ "oxc_span",
+ "oxc_syntax",
+ "rustc-hash",
+ "ryu-js",
+]
+
+[[package]]
+name = "oxc_data_structures"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e902c119eca30d0016d3d2aed31052d6e3ef1f594d71ca3fe193286771a8088d"
+dependencies = [
+ "assert-unchecked",
+ "ropey",
+]
+
+[[package]]
+name = "oxc_diagnostics"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "640a5f40eb0e09725a5e9c90e88727272c999d9805403747e2140cab2a742b6e"
+dependencies = [
+ "cow-utils",
+ "oxc-miette",
+]
+
+[[package]]
+name = "oxc_ecmascript"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87543a59fd0d2d27102ccd8eee9f954564f7e82124992cc21975bbbe75851ce4"
+dependencies = [
+ "cow-utils",
+ "num-bigint",
+ "num-traits",
+ "oxc_ast",
+ "oxc_span",
+ "oxc_syntax",
+]
+
+[[package]]
+name = "oxc_estree"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1efeb3bb8ce6bced1e382e49d3978a2e5067cdc3881584f1cdef5aaf52d28b4"
+dependencies = [
+ "itoa",
+]
+
+[[package]]
+name = "oxc_index"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5eca5d9726cd0a6e433debe003b7bc88b2ecad0bb6109f0cef7c55e692139a34"
+
+[[package]]
+name = "oxc_mangler"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74396fefe8eb5a62b2d79023b468f3012d0daa25b4a6fdff64816ade625a6dfc"
+dependencies = [
+ "fixedbitset",
+ "itertools",
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_index",
+ "oxc_semantic",
+ "oxc_span",
+ "rustc-hash",
+]
+
+[[package]]
+name = "oxc_minifier"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc9099e8cb43a9362ffde2cfcb5d69bbbf87f0303c36d6209379b4f88f3d866"
+dependencies = [
+ "cow-utils",
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_codegen",
+ "oxc_data_structures",
+ "oxc_ecmascript",
+ "oxc_mangler",
+ "oxc_parser",
+ "oxc_semantic",
+ "oxc_span",
+ "oxc_syntax",
+ "oxc_traverse",
+ "rustc-hash",
+]
+
+[[package]]
+name = "oxc_parser"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0f5195b60a980baf22e34c801c8a37901ee21149e523d2a2fc68111991dd0b6"
+dependencies = [
+ "assert-unchecked",
+ "bitflags",
+ "cow-utils",
+ "memchr",
+ "num-bigint",
+ "num-traits",
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_diagnostics",
+ "oxc_ecmascript",
+ "oxc_regular_expression",
+ "oxc_span",
+ "oxc_syntax",
+ "rustc-hash",
+ "seq-macro",
+]
+
+[[package]]
+name = "oxc_regular_expression"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6b7199022f1e8a071a118737884cdf75bd3ba869471fd500f00dd4612a4faeb"
+dependencies = [
+ "oxc_allocator",
+ "oxc_ast_macros",
+ "oxc_diagnostics",
+ "oxc_estree",
+ "oxc_span",
+ "phf",
+ "rustc-hash",
+ "unicode-id-start",
+]
+
+[[package]]
+name = "oxc_semantic"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55da371ce5345676e602719ce048eb2343b3e7b1798c6275e06ea431917340af"
+dependencies = [
+ "assert-unchecked",
+ "itertools",
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_cfg",
+ "oxc_data_structures",
+ "oxc_diagnostics",
+ "oxc_ecmascript",
+ "oxc_index",
+ "oxc_span",
+ "oxc_syntax",
+ "phf",
+ "rustc-hash",
+ "self_cell",
+]
+
+[[package]]
+name = "oxc_sourcemap"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcb048fad7eee078b23dc3f3be7aba94eeba596c892c9255fc2646fba232a2ec"
+dependencies = [
+ "base64-simd",
+ "cfg-if",
+ "cow-utils",
+ "rustc-hash",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "oxc_span"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3cbaec227f5bb91e52a04f9028915973cbb7522b123b41af2d3189eb1ce41d8"
+dependencies = [
+ "compact_str",
+ "oxc-miette",
+ "oxc_allocator",
+ "oxc_ast_macros",
+ "oxc_estree",
+]
+
+[[package]]
+name = "oxc_syntax"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91bd42fae0c0e6212ea704ed533df47748002abc0af5496911fdb09735028a77"
+dependencies = [
+ "assert-unchecked",
+ "bitflags",
+ "cow-utils",
+ "nonmax",
+ "oxc_allocator",
+ "oxc_ast_macros",
+ "oxc_estree",
+ "oxc_index",
+ "oxc_span",
+ "phf",
+ "rustc-hash",
+ "ryu-js",
+ "unicode-id-start",
+]
+
+[[package]]
+name = "oxc_traverse"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c3558bebe882232c4c6fd5f28f08ce58586970c56b486a694b29c59b8b976a8"
+dependencies = [
+ "compact_str",
+ "itoa",
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_data_structures",
+ "oxc_ecmascript",
+ "oxc_semantic",
+ "oxc_span",
+ "oxc_syntax",
+ "rustc-hash",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
]
[[package]]
@@ -1300,35 +1675,27 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.7",
+ "redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
-name = "parser"
-version = "0.5.0"
+name = "path-absolutize"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
dependencies = [
- "allocative",
- "bincode",
- "bitcoin_hashes",
- "biter",
- "chrono",
- "clap",
- "color-eyre",
- "ctrlc",
- "derive_deref",
- "inferno",
- "itertools",
- "ordered-float",
- "rayon",
- "reqwest",
- "sanakirja",
- "serde",
- "serde_json",
- "struct_iterable",
- "toml",
- "zstd",
+ "path-dedot",
+]
+
+[[package]]
+name = "path-dedot"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
+dependencies = [
+ "once_cell",
]
[[package]]
@@ -1338,10 +1705,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
-name = "pin-project-lite"
-version = "0.2.14"
+name = "petgraph"
+version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
+checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
+[[package]]
+name = "phf"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
+dependencies = [
+ "phf_macros",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
+dependencies = [
+ "phf_shared",
+ "rand",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
[[package]]
name = "pin-utils"
@@ -1355,38 +1774,54 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
+[[package]]
+name = "portable-atomic"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
+dependencies = [
+ "portable-atomic",
+]
+
[[package]]
name = "ppv-lite86"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
dependencies = [
- "zerocopy",
+ "zerocopy 0.7.35",
]
[[package]]
name = "proc-macro2"
-version = "1.0.89"
+version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
dependencies = [
"unicode-ident",
]
[[package]]
-name = "quick-xml"
-version = "0.26.0"
+name = "quick_cache"
+version = "0.6.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
+checksum = "3f67cfc9c723c39f3615eb0840b00c4cb9e2b068d2fa761a30d845ec91730a59"
dependencies = [
- "memchr",
+ "equivalent",
+ "hashbrown 0.14.5",
]
[[package]]
name = "quote"
-version = "1.0.37"
+version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
dependencies = [
"proc-macro2",
]
@@ -1418,9 +1853,15 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
]
+[[package]]
+name = "rapidhash"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9813f789f95ee4fe6b4d01834404d7cccacbc3f6c029343af910b3c2835eb9f1"
+
[[package]]
name = "rayon"
version = "1.10.0"
@@ -1443,90 +1884,75 @@ dependencies = [
[[package]]
name = "redox_syscall"
-version = "0.2.16"
+version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
]
[[package]]
-name = "redox_syscall"
-version = "0.5.7"
+name = "regex"
+version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
- "bitflags 2.6.0",
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
]
[[package]]
-name = "reqwest"
-version = "0.12.9"
+name = "regex-automata"
+version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
- "base64 0.22.1",
- "bytes",
- "encoding_rs",
- "futures-channel",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "http-body-util",
- "hyper",
- "hyper-rustls",
- "hyper-tls",
- "hyper-util",
- "ipnet",
- "js-sys",
- "log",
- "mime",
- "native-tls",
- "once_cell",
- "percent-encoding",
- "pin-project-lite",
- "rustls-pemfile",
- "serde",
- "serde_json",
- "serde_urlencoded",
- "sync_wrapper",
- "system-configuration",
- "tokio",
- "tokio-native-tls",
- "tower-service",
- "url",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "web-sys",
- "windows-registry",
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
]
[[package]]
-name = "rgb"
-version = "0.8.50"
+name = "regex-syntax"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a"
-dependencies = [
- "bytemuck",
-]
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "ring"
-version = "0.17.8"
+version = "0.17.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73"
dependencies = [
"cc",
"cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
"libc",
- "spin",
"untrusted",
"windows-sys 0.52.0",
]
+[[package]]
+name = "rlimit"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "ropey"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93411e420bcd1a75ddd1dc3caf18c23155eda2c090631a85af21ba19e97093b5"
+dependencies = [
+ "smallvec",
+ "str_indices",
+]
+
[[package]]
name = "rustc-demangle"
version = "0.1.24"
@@ -1534,92 +1960,63 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
-name = "rustix"
-version = "0.38.37"
+name = "rustc-hash"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+
+[[package]]
+name = "rustix"
+version = "0.38.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags",
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
name = "rustls"
-version = "0.23.14"
+version = "0.21.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8"
+checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
dependencies = [
- "once_cell",
- "rustls-pki-types",
+ "log",
+ "ring",
"rustls-webpki",
- "subtle",
- "zeroize",
+ "sct",
]
-[[package]]
-name = "rustls-pemfile"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
-dependencies = [
- "rustls-pki-types",
-]
-
-[[package]]
-name = "rustls-pki-types"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55"
-
[[package]]
name = "rustls-webpki"
-version = "0.102.8"
+version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
"ring",
- "rustls-pki-types",
"untrusted",
]
+[[package]]
+name = "rustversion"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
+
[[package]]
name = "ryu"
-version = "1.0.18"
+version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd"
[[package]]
-name = "sanakirja"
-version = "1.4.3"
+name = "ryu-js"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81aaf70d064e2122209f04d01fd91e8908e7a327b516236e1cbc0c3f34ac6d11"
-dependencies = [
- "fs2",
- "log",
- "memmap2",
- "parking_lot",
- "sanakirja-core",
- "serde",
- "thiserror",
-]
-
-[[package]]
-name = "sanakirja-core"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5"
-
-[[package]]
-name = "schannel"
-version = "0.1.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1"
-dependencies = [
- "windows-sys 0.59.0",
-]
+checksum = "dd29631678d6fb0903b69223673e122c32e9ae559d0960a38d574695ebc0ea15"
[[package]]
name = "scopeguard"
@@ -1627,6 +2024,16 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "sct"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
[[package]]
name = "secp256k1"
version = "0.29.1"
@@ -1649,53 +2056,51 @@ dependencies = [
]
[[package]]
-name = "security-framework"
-version = "2.11.1"
+name = "self_cell"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
-dependencies = [
- "bitflags 2.6.0",
- "core-foundation",
- "core-foundation-sys",
- "libc",
- "security-framework-sys",
-]
+checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe"
[[package]]
-name = "security-framework-sys"
-version = "2.12.0"
+name = "seq-macro"
+version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
+checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
-version = "1.0.214"
+version = "1.0.218"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
+checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60"
dependencies = [
"serde_derive",
]
[[package]]
-name = "serde_derive"
-version = "1.0.214"
+name = "serde_bytes"
+version = "0.11.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
+checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.218"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.85",
+ "syn 2.0.98",
]
[[package]]
name = "serde_json"
-version = "1.0.132"
+version = "1.0.139"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
+checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6"
dependencies = [
"itoa",
"memchr",
@@ -1704,11 +2109,12 @@ dependencies = [
]
[[package]]
-name = "serde_spanned"
-version = "0.6.8"
+name = "serde_path_to_error"
+version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
+checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6"
dependencies = [
+ "itoa",
"serde",
]
@@ -1740,75 +2146,76 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
-name = "slab"
-version = "0.4.9"
+name = "signal-hook-registry"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
dependencies = [
- "autocfg",
+ "libc",
]
[[package]]
-name = "smallvec"
-version = "1.13.2"
+name = "simdutf8"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
+[[package]]
+name = "siphasher"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
+
+[[package]]
+name = "smallvec"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
+
+[[package]]
+name = "smawk"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
[[package]]
name = "socket2"
-version = "0.5.7"
+version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
+checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
-name = "spin"
-version = "0.9.8"
+name = "static_assertions"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
-name = "str_stack"
+name = "std-semaphore"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
+checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
-[[package]]
-name = "struct_iterable"
-version = "0.1.2"
+name = "storable_vec"
+version = "0.1.3"
dependencies = [
- "struct_iterable_derive",
- "struct_iterable_internal",
+ "memmap2",
+ "rayon",
+ "serde",
+ "serde_json",
+ "zerocopy 0.8.20",
]
[[package]]
-name = "struct_iterable_derive"
-version = "0.1.0"
-dependencies = [
- "proc-macro2",
- "quote",
- "struct_iterable_internal",
- "syn 2.0.85",
-]
-
-[[package]]
-name = "struct_iterable_internal"
-version = "0.1.1"
-
-[[package]]
-name = "subtle"
-version = "2.6.1"
+name = "str_indices"
+version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6"
[[package]]
name = "syn"
@@ -1823,9 +2230,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.85"
+version = "2.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56"
+checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1"
dependencies = [
"proc-macro2",
"quote",
@@ -1834,65 +2241,53 @@ dependencies = [
[[package]]
name = "sync_wrapper"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
-dependencies = [
- "futures-core",
-]
-
-[[package]]
-name = "system-configuration"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
-dependencies = [
- "bitflags 2.6.0",
- "core-foundation",
- "system-configuration-sys",
-]
-
-[[package]]
-name = "system-configuration-sys"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
[[package]]
name = "tempfile"
-version = "3.13.0"
+version = "3.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
+checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230"
dependencies = [
"cfg-if",
"fastrand",
+ "getrandom 0.3.1",
"once_cell",
"rustix",
"windows-sys 0.59.0",
]
[[package]]
-name = "thiserror"
-version = "1.0.64"
+name = "textwrap"
+version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
+checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
+dependencies = [
+ "smawk",
+ "unicode-linebreak",
+ "unicode-width 0.1.14",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.64"
+version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.85",
+ "syn 2.0.98",
]
[[package]]
@@ -1905,62 +2300,40 @@ dependencies = [
"once_cell",
]
-[[package]]
-name = "tinyvec"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-
[[package]]
name = "tokio"
-version = "1.40.0"
+version = "1.43.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
+checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
+ "parking_lot",
"pin-project-lite",
+ "signal-hook-registry",
"socket2",
+ "tokio-macros",
"windows-sys 0.52.0",
]
[[package]]
-name = "tokio-native-tls"
-version = "0.3.1"
+name = "tokio-macros"
+version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
- "native-tls",
- "tokio",
-]
-
-[[package]]
-name = "tokio-rustls"
-version = "0.26.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
-dependencies = [
- "rustls",
- "rustls-pki-types",
- "tokio",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
]
[[package]]
name = "tokio-util"
-version = "0.7.12"
+version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
+checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
dependencies = [
"bytes",
"futures-core",
@@ -1970,38 +2343,45 @@ dependencies = [
]
[[package]]
-name = "toml"
-version = "0.8.19"
+name = "tower"
+version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
+checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+ "tracing",
]
[[package]]
-name = "toml_datetime"
-version = "0.6.8"
+name = "tower-http"
+version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697"
dependencies = [
- "serde",
+ "async-compression",
+ "bitflags",
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "pin-project-lite",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
]
[[package]]
-name = "toml_edit"
-version = "0.22.22"
+name = "tower-layer"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
-dependencies = [
- "indexmap",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "winnow",
-]
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
@@ -2011,19 +2391,20 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
-version = "0.1.40"
+version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
+checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
dependencies = [
+ "log",
"pin-project-lite",
"tracing-core",
]
[[package]]
name = "tracing-core"
-version = "0.1.32"
+version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
+checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
dependencies = [
"once_cell",
"valuable",
@@ -2031,9 +2412,9 @@ dependencies = [
[[package]]
name = "tracing-error"
-version = "0.2.0"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
+checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db"
dependencies = [
"tracing",
"tracing-subscriber",
@@ -2041,9 +2422,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
-version = "0.3.18"
+version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
dependencies = [
"sharded-slab",
"thread_local",
@@ -2051,31 +2432,34 @@ dependencies = [
]
[[package]]
-name = "try-lock"
-version = "0.2.5"
+name = "unicode-id-start"
+version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
-
-[[package]]
-name = "unicode-bidi"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893"
+checksum = "2f322b60f6b9736017344fa0635d64be2f458fbc04eef65f6be22976dd1ffd5b"
[[package]]
name = "unicode-ident"
-version = "1.0.13"
+version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
+checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
[[package]]
-name = "unicode-normalization"
-version = "0.1.24"
+name = "unicode-linebreak"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
-dependencies = [
- "tinyvec",
-]
+checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
+
+[[package]]
+name = "unicode-width"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
[[package]]
name = "untrusted"
@@ -2083,23 +2467,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
-[[package]]
-name = "unty"
-version = "0.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129"
-
-[[package]]
-name = "url"
-version = "2.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
-dependencies = [
- "form_urlencoded",
- "idna",
- "percent-encoding",
-]
-
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -2108,37 +2475,39 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "valuable"
-version = "0.1.0"
+version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
-name = "vcpkg"
-version = "0.2.15"
+name = "value-log"
+version = "1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
-[[package]]
-name = "virtue"
-version = "0.0.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7302ac74a033bf17b6e609ceec0f891ca9200d502d31f02dc7908d3d98767c9d"
-
-[[package]]
-name = "want"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+checksum = "f2398750880b2b9770afbd1a3e299b9e859d6143c299867eb35fdf484b7625d3"
dependencies = [
- "try-lock",
+ "byteorder",
+ "byteview",
+ "interval-heap",
+ "log",
+ "path-absolutize",
+ "quick_cache",
+ "rustc-hash",
+ "tempfile",
+ "xxhash-rust",
]
+[[package]]
+name = "varint-rs"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
+
+[[package]]
+name = "vsimd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
+
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
@@ -2146,142 +2515,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
-name = "wasm-bindgen"
-version = "0.2.93"
+name = "wasi"
+version = "0.13.3+wasi-0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
+checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
dependencies = [
- "cfg-if",
- "once_cell",
- "wasm-bindgen-macro",
+ "wit-bindgen-rt",
]
[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.93"
+name = "webpki-roots"
+version = "0.25.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
-dependencies = [
- "bumpalo",
- "log",
- "once_cell",
- "proc-macro2",
- "quote",
- "syn 2.0.85",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-futures"
-version = "0.4.43"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
-dependencies = [
- "cfg-if",
- "js-sys",
- "wasm-bindgen",
- "web-sys",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.93"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.93"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.85",
- "wasm-bindgen-backend",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.93"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
-
-[[package]]
-name = "web-sys"
-version = "0.3.70"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
-dependencies = [
- "js-sys",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows-core"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-registry"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
-dependencies = [
- "windows-result",
- "windows-strings",
- "windows-targets",
-]
-
-[[package]]
-name = "windows-result"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-strings"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
-dependencies = [
- "windows-result",
- "windows-targets",
-]
+checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
[[package]]
name = "windows-sys"
@@ -2366,14 +2612,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
-name = "winnow"
-version = "0.6.20"
+name = "wit-bindgen-rt"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
+checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
dependencies = [
- "memchr",
+ "bitflags",
]
+[[package]]
+name = "xxhash-rust"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+
[[package]]
name = "zerocopy"
version = "0.7.35"
@@ -2381,7 +2633,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"byteorder",
- "zerocopy-derive",
+ "zerocopy-derive 0.7.35",
+]
+
+[[package]]
+name = "zerocopy"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dde3bb8c68a8f3f1ed4ac9221aad6b10cece3e60a8e2ea54a6a2dec806d0084c"
+dependencies = [
+ "zerocopy-derive 0.8.20",
]
[[package]]
@@ -2392,38 +2653,43 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.85",
+ "syn 2.0.98",
]
[[package]]
-name = "zeroize"
-version = "1.8.1"
+name = "zerocopy-derive"
+version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+checksum = "eea57037071898bf96a6da35fd626f4f27e9cee3ead2a6c703cf09d472b2e700"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
[[package]]
name = "zstd"
-version = "0.13.2"
+version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
-version = "7.2.1"
+version = "7.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
+checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
-version = "2.0.13+zstd.1.5.6"
+version = "2.0.14+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
+checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5"
dependencies = [
"cc",
"pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 000000000..48a9425e5
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,31 @@
+[workspace]
+members = ["crates/*"]
+resolver = "2"
+package.license = "MIT"
+package.edition = "2024"
+package.version = "0.0.0"
+
+[workspace.dependencies]
+bitcoin = { version = "0.32.5", features = ["serde"] }
+brk_computer = { version = "0", path = "crates/brk_computer" }
+brk_fetcher = { version = "0", path = "crates/brk_fetcher" }
+brk_indexer = { version = "0", path = "crates/brk_indexer" }
+brk_parser = { version = "0", path = "crates/brk_parser" }
+brk_logger = { version = "0", path = "crates/brk_logger" }
+brk_server = { version = "0", path = "crates/brk_server" }
+byteview = "0.5.4"
+color-eyre = "0.6.3"
+derive_deref = "1.1.1"
+fjall = "2.6.5"
+hodor = { version = "0", path = "crates/hodor" }
+jiff = "0.2.1"
+log = { version = "0.4.26" }
+minreq = { version = "2.13.2", features = ["https", "serde_json"] }
+rayon = "1.10.0"
+serde = { version = "1.0.218", features = ["derive"] }
+serde_bytes = "0.11.15"
+serde_json = { version = "1.0.139", features = ["float_roundtrip"] }
+storable_vec = { version = "0", path = "crates/storable_vec", features = [
+ "json",
+] }
+zerocopy = { version = "0.8.20", features = ["derive"] }
diff --git a/LICENSE.md b/LICENSE.md
index 77197e6d3..4827b5007 100644
--- a/LICENSE.md
+++ b/LICENSE.md
@@ -1,6 +1,6 @@
MIT License
-Copyright (c) 2024 kibō
+Copyright (c) 2025 kibō.money
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/README.md b/README.md
index c2e0ce2cb..7aecf84a9 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
-
-
- 
+
+
+ 
@@ -63,17 +63,18 @@ Please open an issue if you want to add another instance
### Requirements
- At least 16 GB of RAM
-- 1 TB of free space (will use 70% of that without defragmentation and 40% after)
-- A running instance of bitcoin-core with:
- - `-txindex=1`
- - `-blocksxor=0`
- - RPC credentials
- - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly -txindex=1 -blocksxor=0`
+- A disk with 1 TB of free space (will use between 40% to 80% depending on several things)
+ - Recommended: Rated at 3 GB/s (Thunderbolt 4 speed)
+- A running instance of bitcoin-core
+ - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly`
- Git
+- Unix based operating system (Mac OS or Linux)
+ - Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config`
+ - Mac OS:
+ - Disable Spotlight or exclude the `--kibodir` folder from it
+ - Don't use Time Machine or exclude the `--kibodir` folder (especially needed for local snapshots)
-### Manual
-
-_Mac OS and Linux only, Windows is unsupported_
+### Build
First we need to install Rust (https://www.rust-lang.org/tools/install)
@@ -81,74 +82,33 @@ First we need to install Rust (https://www.rust-lang.org/tools/install)
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
```
-If you already had Rust installed you could update it just in case
+If you already had Rust installed you could update it
```bash
rustup update
```
-> If you're on Ubuntu you'll probably also need to install `open-ssl` with
->
-> ```bash
-> sudo apt install libssl-dev pkg-config
-> ```
-
-Optionally, you can also install `cargo-watch` for the server to automatically restart it on file change, which will be triggered by new code and new datasets from the parser (https://github.com/watchexec/cargo-watch?tab=readme-ov-file#install)
-
-```bash
-cargo install cargo-watch --locked
-```
-
-Then you need to choose a path where all files related to **kibō** will live
+Then you need to choose a path where the project will reside and then clone it
```bash
cd ???
-```
-
-We can now clone the repository
-
-```bash
git clone https://github.com/kibo-money/kibo.git
+cd kibo
```
-In a new terminal, go to the `parser`'s folder of the repository
+If it's your first time running kibo, it will need several information such as:
-```bash
-cd ???/kibo/parser
-```
+- `--bitcoindir PATH`: path to bitcoin core data directory, `???/bitcoin`
+- `--kibodir PATH`: path to kibo outputs, if you have enough space on your main disk `~/.kibo` is fine
-Now we can finally start by running the parser, you need to use the `./run.sh` script instead of `cargo run -r` as we need to set various system variables for the program to run smoothly
+Everything will be saved at `~/.kibo/config.toml`, which will allow you to simply run `cargo run -r` next time
-For the first launch, the parser will need several information such as:
-
-- `--datadir`: which is bitcoin data directory path, prefer `$HOME` to `~` as the latter might not work
-
-Optionally you can also specify:
-
-- `--rpccookiefile`: the path to the cookie file if not default
-- `--rpcuser`: the username of the RPC credentials to talk to the bitcoin server if set
-- `--rpcpassword`: the password of the RPC credentials if set
-- `--rpcconnect`: if the bitcoin core server's IP is different than `localhost`
-- `--rpcport`: if the port is different than `8332`
-
-Everything will be saved in a `config.toml` file, which will allow you to simply run `./run.sh` next time
+If you need more options please run `cargo run -r --help` to see what parameters are available.
Here's an example
```bash
-./run.sh --datadir=$HOME/Developer/bitcoin
-```
-
-In a **new** terminal, go to the `server`'s folder of the repository
-
-```bash
-cd ???/kibo/server
-```
-
-And start it also with the `run.sh` script instead of `cargo run -r`
-
-```bash
-./run.sh
+cargo run -r -- --bitcoindir=~/Developer/bitcoin --kibodir=~/.kibo
```
Then the easiest to let others access your server is to use `cloudflared` which will also cache requests. For more information go to: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/
diff --git a/assets/dove-orange.svg b/_assets/dove-orange.svg
similarity index 100%
rename from assets/dove-orange.svg
rename to _assets/dove-orange.svg
diff --git a/assets/dove-white.svg b/_assets/dove-white.svg
similarity index 100%
rename from assets/dove-white.svg
rename to _assets/dove-white.svg
diff --git a/assets/logo-dove-dark.svg b/_assets/logo-dove-dark.svg
similarity index 100%
rename from assets/logo-dove-dark.svg
rename to _assets/logo-dove-dark.svg
diff --git a/assets/logo-dove-light.svg b/_assets/logo-dove-light.svg
similarity index 100%
rename from assets/logo-dove-light.svg
rename to _assets/logo-dove-light.svg
diff --git a/assets/logo-dove-orange.svg b/_assets/logo-dove-orange.svg
similarity index 100%
rename from assets/logo-dove-orange.svg
rename to _assets/logo-dove-orange.svg
diff --git a/assets/logo-full-dark.svg b/_assets/logo-full-dark.svg
similarity index 100%
rename from assets/logo-full-dark.svg
rename to _assets/logo-full-dark.svg
diff --git a/assets/logo-full-light.svg b/_assets/logo-full-light.svg
similarity index 100%
rename from assets/logo-full-light.svg
rename to _assets/logo-full-light.svg
diff --git a/assets/logo-icon.svg b/_assets/logo-icon.svg
similarity index 100%
rename from assets/logo-icon.svg
rename to _assets/logo-icon.svg
diff --git a/assets/logo-long-text-dark.svg b/_assets/logo-long-text-dark.svg
similarity index 100%
rename from assets/logo-long-text-dark.svg
rename to _assets/logo-long-text-dark.svg
diff --git a/assets/logo-long-text-light.svg b/_assets/logo-long-text-light.svg
similarity index 100%
rename from assets/logo-long-text-light.svg
rename to _assets/logo-long-text-light.svg
diff --git a/assets/logo-short-text-dark.svg b/_assets/logo-short-text-dark.svg
similarity index 100%
rename from assets/logo-short-text-dark.svg
rename to _assets/logo-short-text-dark.svg
diff --git a/assets/logo-short-text-light.svg b/_assets/logo-short-text-light.svg
similarity index 100%
rename from assets/logo-short-text-light.svg
rename to _assets/logo-short-text-light.svg
diff --git a/assets/logo-stamp-orange.svg b/_assets/logo-stamp-orange.svg
similarity index 100%
rename from assets/logo-stamp-orange.svg
rename to _assets/logo-stamp-orange.svg
diff --git a/assets/logo-stamp.svg b/_assets/logo-stamp.svg
similarity index 100%
rename from assets/logo-stamp.svg
rename to _assets/logo-stamp.svg
diff --git a/assets/v0.0.X.jpg b/_assets/v0.0.X.jpg
similarity index 100%
rename from assets/v0.0.X.jpg
rename to _assets/v0.0.X.jpg
diff --git a/assets/v0.1.0.jpg b/_assets/v0.1.0.jpg
similarity index 100%
rename from assets/v0.1.0.jpg
rename to _assets/v0.1.0.jpg
diff --git a/assets/v0.1.1.jpg b/_assets/v0.1.1.jpg
similarity index 100%
rename from assets/v0.1.1.jpg
rename to _assets/v0.1.1.jpg
diff --git a/assets/v0.2.0.jpg b/_assets/v0.2.0.jpg
similarity index 100%
rename from assets/v0.2.0.jpg
rename to _assets/v0.2.0.jpg
diff --git a/assets/v0.3.0.jpg b/_assets/v0.3.0.jpg
similarity index 100%
rename from assets/v0.3.0.jpg
rename to _assets/v0.3.0.jpg
diff --git a/assets/v0.4.0.jpg b/_assets/v0.4.0.jpg
similarity index 100%
rename from assets/v0.4.0.jpg
rename to _assets/v0.4.0.jpg
diff --git a/assets/v0.5.0.jpg b/_assets/v0.5.0.jpg
similarity index 100%
rename from assets/v0.5.0.jpg
rename to _assets/v0.5.0.jpg
diff --git a/docker/Dockerfile b/_docker/Dockerfile
similarity index 100%
rename from docker/Dockerfile
rename to _docker/Dockerfile
diff --git a/docker/build.sh b/_docker/build.sh
similarity index 100%
rename from docker/build.sh
rename to _docker/build.sh
diff --git a/docker/cmd.sh b/_docker/cmd.sh
similarity index 100%
rename from docker/cmd.sh
rename to _docker/cmd.sh
diff --git a/docker/run.sh b/_docker/run.sh
similarity index 100%
rename from docker/run.sh
rename to _docker/run.sh
diff --git a/_src/main.rs b/_src/main.rs
new file mode 100644
index 000000000..3dfe00316
--- /dev/null
+++ b/_src/main.rs
@@ -0,0 +1,67 @@
+use brk_parser::bitcoincore_rpc::Client;
+use log::info;
+use rlimit::{Resource, getrlimit, setrlimit};
+
+mod io;
+mod parser;
+mod server;
+mod structs;
+mod utils;
+
+use brk_parser::Datasets;
+use server::api::structs::Routes;
+use structs::{Config, Exit};
+use utils::init_log;
+
+fn main() -> color_eyre::Result<()> {
+ color_eyre::install()?;
+
+ init_log();
+
+ let (_, nofile_limit) = getrlimit(Resource::NOFILE).unwrap();
+ setrlimit(Resource::NOFILE, 138_240, nofile_limit)?;
+
+ std::thread::Builder::new()
+ .stack_size(getrlimit(Resource::STACK).unwrap().1 as usize)
+ .spawn(|| -> color_eyre::Result<()> {
+ let exit = Exit::new();
+
+ let config = Config::import()?;
+
+ info!("Starting...");
+
+ let rpc = Client::from(&config);
+
+ let routes = Routes::build(&Datasets::import(&config)?, &config);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async {
+ let run_parser = config.parser();
+ let run_server = config.server();
+
+ let config_clone = config.clone();
+ let handle = tokio::spawn(async move {
+ if run_server {
+ server::main(routes, config_clone).await.unwrap();
+ } else {
+ info!("Skipping server");
+ }
+ });
+
+ if run_parser {
+ parser::main(&config, &rpc, &exit)?;
+ } else {
+ info!("Skipping parser");
+ }
+
+ handle.await?;
+
+ Ok(())
+ })
+ })?
+ .join()
+ .unwrap()
+}
diff --git a/_src/parser/actions/export.rs b/_src/parser/actions/export.rs
new file mode 100644
index 000000000..5a82e71a3
--- /dev/null
+++ b/_src/parser/actions/export.rs
@@ -0,0 +1,63 @@
+use log::info;
+
+use crate::{
+ parser::{databases::Databases, datasets::Datasets, states::States},
+ structs::{Config, Date, Exit, Height},
+ utils::time,
+};
+
+pub struct ExportedData<'a> {
+ pub config: &'a Config,
+ pub databases: Option<&'a mut Databases>,
+ pub datasets: &'a mut Datasets,
+ pub date: Date,
+ pub defragment: bool,
+ pub exit: Exit,
+ pub height: Height,
+ pub states: Option<&'a States>,
+}
+
+pub fn export(
+ ExportedData {
+ config,
+ databases,
+ datasets,
+ date,
+ defragment,
+ exit,
+ height,
+ states,
+ }: ExportedData,
+) -> color_eyre::Result<()> {
+ if exit.active() {
+ info!("Exit in progress, skipping export");
+ return Ok(());
+ }
+
+ exit.block();
+
+ let text = if defragment {
+ "Exporting and defragmenting..."
+ } else {
+ "Exporting..."
+ };
+ info!("{text}");
+
+ time("Finished export", || -> color_eyre::Result<()> {
+ datasets.export(config, height)?;
+
+ if let Some(databases) = databases {
+ databases.export(height, date, defragment)?;
+ }
+
+ if let Some(states) = states {
+ states.export(config)?;
+ }
+
+ Ok(())
+ })?;
+
+ exit.unblock();
+
+ Ok(())
+}
diff --git a/parser/src/actions/iter_blocks.rs b/_src/parser/actions/iter_blocks.rs
similarity index 62%
rename from parser/src/actions/iter_blocks.rs
rename to _src/parser/actions/iter_blocks.rs
index aaf46e35e..bb8eb84be 100644
--- a/parser/src/actions/iter_blocks.rs
+++ b/_src/parser/actions/iter_blocks.rs
@@ -1,70 +1,56 @@
use std::{collections::BTreeSet, time::Instant};
+use brk_parser::bitcoincore_rpc::Client;
+use chrono::Datelike;
use export::ExportedData;
use itertools::Itertools;
+use log::info;
use parse::ParseData;
use crate::{
- actions::{export, find_first_inserted_unsafe_height, parse},
- create_rpc,
- databases::Databases,
- datasets::{AllDatasets, ComputeData},
- io::OUTPUTS_FOLDER_PATH,
- states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
- structs::{DateData, MapKey, Timestamp},
- utils::{generate_allocation_files, log, time},
- Config, Exit, Height,
+ parser::{
+ actions::{export, find_first_inserted_unsafe_height, parse},
+ databases::Databases,
+ datasets::{ComputeData, Datasets},
+ states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
+ },
+ structs::{Config, DateData, DisplayInstant, Exit, Height, MapKey, Timestamp},
+ utils::{generate_allocation_files, time},
};
pub fn iter_blocks(
- config: &mut Config,
- rpc: &biter::bitcoincore_rpc::Client,
+ config: &Config,
+ rpc: &Client,
approx_block_count: usize,
exit: Exit,
+ databases: &mut Databases,
+ datasets: &mut Datasets,
) -> color_eyre::Result<()> {
- log("Starting...");
+ let mut states = States::import(config).unwrap_or_default();
- let mut datasets = AllDatasets::import(config)?;
+ info!("Imported states");
- log("Imported datasets");
-
- let mut databases = Databases::import();
-
- if config.first_defragment() {
- databases.defragment(&exit);
- config.disable_defragment();
- }
-
- log("Imported databases");
-
- let mut states = States::import().unwrap_or_default();
-
- log("Imported states");
-
- let first_unsafe_heights =
- find_first_inserted_unsafe_height(&mut states, &mut databases, &mut datasets);
+ let first_unsafe_heights = find_first_inserted_unsafe_height(&mut states, databases, datasets, config);
let mut height = first_unsafe_heights.min();
- log(&format!("Starting parsing at height: {height}"));
+ info!("Starting parsing at height: {height}");
let mut next_block_opt = None;
let mut blocks_loop_date = None;
+ let mut next_date_opt;
let block_receiver = biter::new(
- config.datadir.as_ref().unwrap(),
- OUTPUTS_FOLDER_PATH,
+ &config.path_bitcoindir(),
Some(height.to_usize()),
None,
- create_rpc(config).unwrap(),
+ Client::from(config),
);
let mut block_iter = block_receiver.iter();
'parsing: loop {
- let instant = Instant::now();
-
let mut processed_heights = BTreeSet::new();
let mut processed_dates = BTreeSet::new();
@@ -75,15 +61,15 @@ pub fn iter_blocks(
blocks_loop_date.take();
}
+ let instant = Instant::now();
+
'blocks: loop {
let current_block_opt = next_block_opt.take().or_else(|| block_iter.next());
next_block_opt = block_iter.next();
- if let Some((_current_block_height, current_block, _current_block_hash)) =
- current_block_opt
- {
- let timestamp = Timestamp::wrap(current_block.header.time);
+ if let Some((_current_block_height, current_block, _current_block_hash)) = current_block_opt {
+ let timestamp = Timestamp::from(current_block.header.time);
let current_block_date = timestamp.to_date();
let current_block_height: Height = height + blocks_loop_i;
@@ -93,16 +79,12 @@ pub fn iter_blocks(
panic!()
}
- let next_block_date = next_block_opt.as_ref().map(|(_, next_block, _)| {
- Timestamp::wrap(next_block.header.time).to_date()
- });
+ next_date_opt = next_block_opt
+ .as_ref()
+ .map(|(_, next_block, _)| Timestamp::from(next_block.header.time).to_date());
// Always run for the first block of the loop
if blocks_loop_date.is_none() {
- log(&format!(
- "Processing {current_block_date} (height: {height})..."
- ));
-
blocks_loop_date.replace(current_block_date);
if states
@@ -111,9 +93,7 @@ pub fn iter_blocks(
.map(|date_data| *date_data.date < *current_block_date)
.unwrap_or(true)
{
- states
- .date_data_vec
- .push(DateData::new(current_block_date, vec![]));
+ states.date_data_vec.push(DateData::new(current_block_date, vec![]));
}
processed_dates.insert(current_block_date);
@@ -125,17 +105,15 @@ pub fn iter_blocks(
panic!("current block should always have the same date as the current blocks loop");
}
- let is_date_last_block = next_block_date
+ let is_date_last_block = next_date_opt
// Do NOT change `blocks_loop_date` to `current_block_date` !!!
.map_or(true, |next_block_date| blocks_loop_date < next_block_date);
processed_heights.insert(current_block_height);
if first_unsafe_heights.inserted <= current_block_height {
- let compute_addresses = databases.check_if_needs_to_compute_addresses(
- current_block_height,
- blocks_loop_date,
- );
+ let compute_addresses =
+ databases.check_if_needs_to_compute_addresses(current_block_height, blocks_loop_date);
if states.address_cohorts_durable_states.is_none()
&& (compute_addresses
@@ -143,10 +121,9 @@ pub fn iter_blocks(
.address
.needs_durable_states(current_block_height, current_block_date))
{
- states.address_cohorts_durable_states =
- Some(AddressCohortsDurableStates::init(
- &mut databases.address_index_to_address_data,
- ));
+ states.address_cohorts_durable_states = Some(AddressCohortsDurableStates::init(
+ &mut databases.address_index_to_address_data,
+ ));
}
if states.utxo_cohorts_durable_states.is_none()
@@ -159,16 +136,17 @@ pub fn iter_blocks(
}
parse(ParseData {
- rpc,
block: current_block,
block_index: blocks_loop_i,
compute_addresses,
- databases: &mut databases,
- datasets: &mut datasets,
+ config,
+ databases,
+ datasets,
date: blocks_loop_date,
first_date_height: height,
height: current_block_height,
is_date_last_block,
+ rpc,
states: &mut states,
});
}
@@ -176,15 +154,16 @@ pub fn iter_blocks(
blocks_loop_i += 1;
if is_date_last_block {
+ info!(
+ "Processed {current_block_date} ({height} - {current_block_height}) {}",
+ instant.display()
+ );
+
height += blocks_loop_i;
- let is_check_point = next_block_date
- .as_ref()
- .map_or(true, |date| date.is_first_of_month());
+ let is_check_point = next_date_opt.as_ref().map_or(true, |date| date.is_first_of_month());
- let ran_for_at_least_a_minute = instant.elapsed().as_secs() >= 60;
-
- if (is_check_point && ran_for_at_least_a_minute)
+ if (is_check_point && instant.elapsed().as_secs() >= 1)
|| height.is_close_to_end(approx_block_count)
{
break 'days;
@@ -201,14 +180,9 @@ pub fn iter_blocks(
// Don't remember why -1
let last_height = height - 1_u32;
- log(&format!(
- "Parsing group took {} seconds (last height: {last_height})\n",
- instant.elapsed().as_secs_f32(),
- ));
-
if first_unsafe_heights.computed <= last_height {
- log("Computing datasets...");
- time("Computing datasets", || {
+ info!("Computing datasets...");
+ time("Computed datasets", || {
let dates = processed_dates.into_iter().collect_vec();
let heights = processed_heights.into_iter().collect_vec();
@@ -223,25 +197,31 @@ pub fn iter_blocks(
if !config.dry_run() {
let is_safe = height.is_safe(approx_block_count);
+ let defragment = is_safe
+ && next_date_opt.is_some_and(|date| {
+ (date.year() >= 2020 && date.is_january() || date.year() >= 2022 && date.is_july())
+ && date.is_first_of_month()
+ });
+
export(ExportedData {
- databases: is_safe.then_some(&mut databases),
- datasets: &mut datasets,
+ config,
+ databases: is_safe.then_some(databases),
+ datasets,
date: blocks_loop_date.unwrap(),
+ defragment,
height: last_height,
states: is_safe.then_some(&states),
exit: exit.clone(),
})?;
if config.record_ram_usage() {
- time("Exporing allocation files", || {
- generate_allocation_files(&datasets, &databases, &states, last_height)
+ time("Exporting allocation files", || {
+ generate_allocation_files(datasets, databases, &states, last_height)
})?;
}
} else {
- log("Skipping export");
+ info!("Skipping export");
}
-
- println!();
}
Ok(())
diff --git a/parser/src/actions/min_height.rs b/_src/parser/actions/min_height.rs
similarity index 82%
rename from parser/src/actions/min_height.rs
rename to _src/parser/actions/min_height.rs
index 953c1ddf1..e71b6ae50 100644
--- a/parser/src/actions/min_height.rs
+++ b/_src/parser/actions/min_height.rs
@@ -1,9 +1,12 @@
+use log::info;
+
use crate::{
- databases::Databases,
- datasets::{AllDatasets, AnyDatasets},
- states::States,
- structs::Height,
- utils::log,
+ parser::{
+ databases::Databases,
+ datasets::{AnyDatasets, Datasets},
+ states::States,
+ },
+ structs::{Config, Height},
};
#[derive(Default, Debug)]
@@ -21,7 +24,8 @@ impl Heights {
pub fn find_first_inserted_unsafe_height(
states: &mut States,
databases: &mut Databases,
- datasets: &mut AllDatasets,
+ datasets: &mut Datasets,
+ config: &Config,
) -> Heights {
let min_initial_inserted_last_address_height = datasets
.address
@@ -51,7 +55,7 @@ pub fn find_first_inserted_unsafe_height(
.map(|date_data| date_data.date)
.and_then(|last_safe_date| {
if !usable_databases {
- log("Unusable databases");
+ info!("Unusable databases");
return None;
}
@@ -61,8 +65,8 @@ pub fn find_first_inserted_unsafe_height(
let min_datasets_inserted_last_height = datasets_min_initial_states.inserted.last_height;
let min_datasets_inserted_last_date = datasets_min_initial_states.inserted.last_date;
- log(&format!("min_datasets_inserted_last_height: {:?}", min_datasets_inserted_last_height));
- log(&format!("min_datasets_inserted_last_date: {:?}", min_datasets_inserted_last_date));
+ info!("min_datasets_inserted_last_height: {:?}", min_datasets_inserted_last_height);
+ info!("min_datasets_inserted_last_date: {:?}", min_datasets_inserted_last_date);
let inserted_last_date_is_older_than_saved_state = min_datasets_inserted_last_date.map_or(true, |min_datasets_last_date| min_datasets_last_date < last_safe_date);
@@ -80,7 +84,7 @@ pub fn find_first_inserted_unsafe_height(
let inserted_heights_and_dates_are_out_of_sync = min_datasets_inserted_last_height.map_or(true, |min_datasets_inserted_last_height| min_datasets_inserted_last_height < last_safe_height);
if inserted_heights_and_dates_are_out_of_sync {
- log(&format!("last_safe_height ({last_safe_height}) > min_datasets_height ({min_datasets_inserted_last_height:?})"));
+ info!("last_safe_height ({last_safe_height}) > min_datasets_height ({min_datasets_inserted_last_height:?})");
None
} else {
@@ -108,18 +112,13 @@ pub fn find_first_inserted_unsafe_height(
)
})
.unwrap_or_else(|| {
- log("Starting over...");
+ info!("Starting over...");
let include_addresses = !usable_databases
|| min_initial_inserted_last_address_date.is_none()
|| min_initial_inserted_last_address_height.is_none();
- // if include_addresses {
- // dbg!(include_addresses);
- // panic!("");
- // }
-
- states.reset(include_addresses);
+ states.reset(config, include_addresses);
databases.reset(include_addresses);
diff --git a/parser/src/actions/mod.rs b/_src/parser/actions/mod.rs
similarity index 100%
rename from parser/src/actions/mod.rs
rename to _src/parser/actions/mod.rs
diff --git a/parser/src/actions/parse.rs b/_src/parser/actions/parse.rs
similarity index 52%
rename from parser/src/actions/parse.rs
rename to _src/parser/actions/parse.rs
index 2199a5c3b..d601ac62c 100644
--- a/parser/src/actions/parse.rs
+++ b/_src/parser/actions/parse.rs
@@ -1,6 +1,6 @@
use std::{collections::BTreeMap, ops::ControlFlow, thread};
-use biter::{
+use brk_parser::{
bitcoin::{Block, Txid},
bitcoincore_rpc::RpcApi,
};
@@ -9,17 +9,19 @@ use itertools::Itertools;
use rayon::prelude::*;
use crate::{
- databases::{
- AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex,
- Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount,
- },
- datasets::{AllDatasets, InsertData},
- states::{
- AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates,
- States, UTXOCohortsOneShotStates, UTXOCohortsSentStates,
+ parser::{
+ databases::{
+ AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex, Databases, TxidToTxData,
+ TxoutIndexToAddressIndex, TxoutIndexToAmount,
+ },
+ datasets::{Datasets, InsertData},
+ states::{
+ AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates, States,
+ UTXOCohortsOneShotStates, UTXOCohortsSentStates,
+ },
},
structs::{
- Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Counter, Date,
+ Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Config, Counter, Date,
EmptyAddressData, Height, PartialTxoutData, Price, SentData, Timestamp, TxData, TxoutIndex,
},
};
@@ -28,9 +30,10 @@ pub struct ParseData<'a> {
// pub bitcoin_cli: &'a BitcoinCli,
pub block: Block,
pub block_index: usize,
+ pub config: &'a Config,
pub compute_addresses: bool,
pub databases: &'a mut Databases,
- pub datasets: &'a mut AllDatasets,
+ pub datasets: &'a mut Datasets,
pub date: Date,
pub first_date_height: Height,
pub height: Height,
@@ -43,6 +46,7 @@ pub fn parse(
ParseData {
block,
block_index,
+ config,
compute_addresses,
databases,
datasets,
@@ -56,7 +60,7 @@ pub fn parse(
) {
// log(&format!("{height}"));
- let timestamp = Timestamp::wrap(block.header.time);
+ let timestamp = Timestamp::from(block.header.time);
// If false, expect that the code is flawless
// or create a 0 value txid database
@@ -72,7 +76,7 @@ pub fn parse(
let block_price = Price::from_dollar(
datasets
.price
- .get_height_ohlc(height, timestamp, previous_timestamp)
+ .get_height_ohlc(height, timestamp, previous_timestamp, config)
.unwrap_or_else(|_| panic!("Expect {height} to have a price"))
.close as f64,
);
@@ -106,8 +110,7 @@ pub fn parse(
let mut block_path_to_sent_data: BTreeMap = BTreeMap::default();
// let mut received_data: ReceivedData = ReceivedData::default();
- let mut address_index_to_address_realized_data: BTreeMap =
- BTreeMap::default();
+ let mut address_index_to_address_realized_data: BTreeMap = BTreeMap::default();
let mut coinbase = Amount::ZERO;
let mut satblocks_destroyed = Amount::ZERO;
@@ -166,263 +169,228 @@ pub fn parse(
)
});
- block
- .txdata
- .iter()
- .enumerate()
- .try_for_each(|(block_tx_index, tx)| {
- let txid = tx.compute_txid();
- let tx_index = databases.txid_to_tx_data.metadata.serial as u32;
+ block.txdata.iter().enumerate().try_for_each(|(block_tx_index, tx)| {
+ let txid = tx.compute_txid();
+ let tx_index = databases.txid_to_tx_data.metadata.serial as u32;
- transaction_count += 1;
+ transaction_count += 1;
- // --
- // outputs
- // ---
+ // --
+ // outputs
+ // ---
- let mut utxos = BTreeMap::new();
- let mut spendable_amount = Amount::ZERO;
+ let mut utxos = BTreeMap::new();
+ let mut spendable_amount = Amount::ZERO;
- let is_coinbase = tx.is_coinbase();
+ let is_coinbase = tx.is_coinbase();
- if is_coinbase != (block_tx_index == 0) {
- unreachable!();
- }
+ if is_coinbase != (block_tx_index == 0) {
+ unreachable!();
+ }
- let mut inputs_sum = Amount::ZERO;
- let mut outputs_sum = Amount::ZERO;
+ let mut inputs_sum = Amount::ZERO;
+ let mut outputs_sum = Amount::ZERO;
- let last_block = states.date_data_vec.last_mut_block().unwrap();
+ let last_block = states.date_data_vec.last_mut_block().unwrap();
- // Before `input` to cover outputs being used in the same block as inputs
- tx.output
- .iter()
- .enumerate()
- .filter_map(|(vout, tx_out)| {
- if vout > (u16::MAX as usize) {
- panic!("vout can indeed be bigger than u16::MAX !");
- }
+ // Before `input` to cover outputs being used in the same block as inputs
+ tx.output
+ .iter()
+ .enumerate()
+ .filter_map(|(vout, tx_out)| {
+ if vout > (u16::MAX as usize) {
+ panic!("vout can indeed be bigger than u16::MAX !");
+ }
- let amount = Amount::wrap(tx_out.value);
+ let amount = Amount::wrap(tx_out.value);
- if is_coinbase {
- coinbase += amount;
- } else {
- outputs_sum += amount;
- }
+ if is_coinbase {
+ coinbase += amount;
+ } else {
+ outputs_sum += amount;
+ }
- partial_txout_data_vec
- .pop()
- .unwrap()
- // None if not worth parsing (empty/op_return/...)
- .map(|partial_txout_data| (vout, partial_txout_data))
- })
- .for_each(|(vout, partial_txout_data)| {
- let vout = vout as u16;
+ partial_txout_data_vec
+ .pop()
+ .unwrap()
+ // None if not worth parsing (empty/op_return/...)
+ .map(|partial_txout_data| (vout, partial_txout_data))
+ })
+ .for_each(|(vout, partial_txout_data)| {
+ let vout = vout as u16;
- let txout_index = TxoutIndex::new(tx_index, vout);
+ let txout_index = TxoutIndex::new(tx_index, vout);
- let PartialTxoutData {
- address,
- address_index_opt,
- amount,
- } = partial_txout_data;
+ let PartialTxoutData {
+ address,
+ address_index_opt,
+ amount,
+ } = partial_txout_data;
- spendable_amount += amount;
+ spendable_amount += amount;
- last_block.receive(amount);
+ last_block.receive(amount);
- utxos.insert(vout, amount);
+ utxos.insert(vout, amount);
+
+ databases.txout_index_to_amount.insert_to_ram(txout_index, amount);
+
+ if compute_addresses {
+ let address = address.unwrap();
+
+ let address_index_to_address_data = address_index_to_address_data.as_mut().unwrap();
+
+ let (address_data, address_index) = {
+ if let Some(address_index) = address_index_opt
+ .or_else(|| databases.address_to_address_index.get_from_ram(&address).cloned())
+ {
+ let address_data = address_index_to_address_data.get_mut(&address_index).unwrap();
+
+ (address_data, address_index)
+ } else {
+ let address_index = databases.address_to_address_index.metadata.serial as u32;
+
+ let address_type = address.to_type();
+
+ if let Some(previous) = databases.address_to_address_index.insert(address, address_index) {
+ dbg!(previous);
+ panic!("address #{address_index} shouldn't be present during put");
+ }
+
+ // Checked new
+ let address_data = address_index_to_address_data
+ .entry(address_index)
+ .and_modify(|_| {
+ panic!("Shouldn't exist");
+ })
+ // Will always insert, it's to avoid insert + get
+ .or_insert(AddressData::new(address_type));
+
+ (address_data, address_index)
+ }
+ };
+
+ // MUST be before received !
+ let address_realized_data = address_index_to_address_realized_data
+ .entry(address_index)
+ .or_insert_with(|| AddressRealizedData::default(address_data));
+
+ address_data.receive(amount, block_price);
+
+ address_realized_data.receive(amount);
databases
- .txout_index_to_amount
- .unsafe_insert(txout_index, amount);
+ .txout_index_to_address_index
+ .insert_to_ram(txout_index, address_index);
+ }
+ });
- if compute_addresses {
- let address = address.unwrap();
+ if !utxos.is_empty() {
+ databases.txid_to_tx_data.insert(
+ &txid,
+ TxData::new(
+ tx_index,
+ BlockPath::new(date_index as u16, block_index as u16),
+ utxos.len() as u16,
+ ),
+ );
+ }
- let address_index_to_address_data =
- address_index_to_address_data.as_mut().unwrap();
+ // ---
+ // inputs
+ // ---
- let (address_data, address_index) = {
- if let Some(address_index) = address_index_opt.or_else(|| {
- databases
- .address_to_address_index
- .unsafe_get_from_puts(&address)
- .cloned()
- }) {
- let address_data = address_index_to_address_data
- .get_mut(&address_index)
- .unwrap();
+ if !is_coinbase {
+ tx.input.iter().try_for_each(|txin| {
+ let outpoint = txin.previous_output;
+ let input_txid = outpoint.txid;
+ let input_vout = outpoint.vout;
- (address_data, address_index)
- } else {
- let address_index =
- databases.address_to_address_index.metadata.serial as u32;
+ let remove_tx_data_from_cached_puts = {
+ let mut is_tx_data_from_cached_puts = false;
- let address_type = address.to_type();
+ let input_tx_data = txid_to_tx_data.get_mut(&input_txid).unwrap().as_mut().or_else(|| {
+ is_tx_data_from_cached_puts = true;
- if let Some(previous) = databases
- .address_to_address_index
- .insert(address, address_index)
- {
- dbg!(previous);
- panic!(
- "address #{address_index} shouldn't be present during put"
- );
- }
+ databases.txid_to_tx_data.get_mut_from_ram(&input_txid)
+ });
- // Checked new
- let address_data = address_index_to_address_data
- .entry(address_index)
- .and_modify(|_| {
- panic!("Shouldn't exist");
- })
- // Will always insert, it's to avoid insert + get
- .or_insert(AddressData::new(address_type));
+ // Can be none because 0 sats inputs happen
+ // https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
- (address_data, address_index)
- }
- };
+ if input_tx_data.is_none() {
+ if !enable_check_if_txout_value_is_zero_in_db
+ || rpc
+ .get_raw_transaction(&input_txid, None)
+ .unwrap()
+ .output
+ .get(input_vout as usize)
+ .unwrap()
+ .value
+ .to_sat()
+ == 0
+ {
+ return ControlFlow::Continue::<()>(());
+ }
- // MUST be before received !
- let address_realized_data = address_index_to_address_realized_data
- .entry(address_index)
- .or_insert_with(|| AddressRealizedData::default(address_data));
-
- address_data.receive(amount, block_price);
-
- address_realized_data.receive(amount);
-
- databases
- .txout_index_to_address_index
- .unsafe_insert(txout_index, address_index);
+ dbg!((input_txid, txid, tx_index, input_vout));
+ panic!("Txid to be in txid_to_tx_data");
}
- });
- if !utxos.is_empty() {
- databases.txid_to_tx_data.insert(
- &txid,
- TxData::new(
- tx_index,
- BlockPath::new(date_index as u16, block_index as u16),
- utxos.len() as u16,
- ),
- );
- }
+ let input_tx_data = input_tx_data.unwrap();
+ let input_tx_index = input_tx_data.index;
+ let input_vout = input_vout as u16;
+ let input_txout_index = TxoutIndex::new(input_tx_index, input_vout);
- // ---
- // inputs
- // ---
+ // if input_tx_index == 2516 || input_tx_index == 2490 {
+ // dbg!(input_tx_index, &input_tx_data.utxos);
+ // }
- if !is_coinbase {
- tx.input.iter().try_for_each(|txin| {
- let outpoint = txin.previous_output;
- let input_txid = outpoint.txid;
- let input_vout = outpoint.vout;
+ // let input_amount = input_tx_data.utxos.remove(&input_vout);
- let remove_tx_data_from_cached_puts = {
- let mut is_tx_data_from_cached_puts = false;
+ let input_amount_and_address_index = databases
+ .txout_index_to_amount
+ .remove(&input_txout_index)
+ .map(|amount| {
+ (
+ amount,
+ databases.txout_index_to_address_index.remove(&input_txout_index),
+ )
+ }) // Remove from cached puts
+ .or_else(|| txout_index_to_amount_and_address_index.remove(&input_txout_index));
- let input_tx_data = txid_to_tx_data
- .get_mut(&input_txid)
- .unwrap()
- .as_mut()
- .or_else(|| {
- is_tx_data_from_cached_puts = true;
-
- databases
- .txid_to_tx_data
- .unsafe_get_mut_from_puts(&input_txid)
- });
-
- // Can be none because 0 sats inputs happen
- // https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
-
- if input_tx_data.is_none() {
- if !enable_check_if_txout_value_is_zero_in_db
- || rpc
- .get_raw_transaction(&input_txid, None)
- .unwrap()
- .output
- .get(input_vout as usize)
- .unwrap()
- .value
- .to_sat()
- == 0
- {
- return ControlFlow::Continue::<()>(());
- }
-
- dbg!((input_txid, txid, tx_index, input_vout));
- panic!("Txid to be in txid_to_tx_data");
+ if input_amount_and_address_index.is_none() {
+ if !enable_check_if_txout_value_is_zero_in_db
+ || rpc
+ .get_raw_transaction(&input_txid, None)
+ .unwrap()
+ .output
+ .get(input_vout as usize)
+ .unwrap()
+ .value
+ .to_sat()
+ == 0
+ {
+ return ControlFlow::Continue::<()>(());
}
- let input_tx_data = input_tx_data.unwrap();
- let input_tx_index = input_tx_data.index;
- let input_vout = input_vout as u16;
- let input_txout_index = TxoutIndex::new(input_tx_index, input_vout);
+ dbg!((input_txid, tx_index, input_tx_index, input_vout, input_tx_data, txid,));
+ panic!("Txout index to be in txout_index_to_txout_value");
+ }
- // if input_tx_index == 2516 || input_tx_index == 2490 {
- // dbg!(input_tx_index, &input_tx_data.utxos);
- // }
+ input_tx_data.utxos -= 1;
- // let input_amount = input_tx_data.utxos.remove(&input_vout);
+ let (input_amount, input_address_index) = input_amount_and_address_index.unwrap();
- let input_amount_and_address_index = databases
- .txout_index_to_amount
- .remove(&input_txout_index)
- .map(|amount| {
- (
- amount,
- databases
- .txout_index_to_address_index
- .remove(&input_txout_index),
- )
- }) // Remove from cached puts
- .or_else(|| {
- txout_index_to_amount_and_address_index.remove(&input_txout_index)
- });
+ let input_block_path = input_tx_data.block_path;
- if input_amount_and_address_index.is_none() {
- if !enable_check_if_txout_value_is_zero_in_db
- || rpc
- .get_raw_transaction(&input_txid, None)
- .unwrap()
- .output
- .get(input_vout as usize)
- .unwrap()
- .value
- .to_sat()
- == 0
- {
- return ControlFlow::Continue::<()>(());
- }
+ let BlockPath {
+ date_index: input_date_index,
+ block_index: input_block_index,
+ } = input_block_path;
- dbg!((
- input_txid,
- tx_index,
- input_tx_index,
- input_vout,
- input_tx_data,
- txid,
- ));
- panic!("Txout index to be in txout_index_to_txout_value");
- }
-
- input_tx_data.utxos -= 1;
-
- let (input_amount, input_address_index) =
- input_amount_and_address_index.unwrap();
-
- let input_block_path = input_tx_data.block_path;
-
- let BlockPath {
- date_index: input_date_index,
- block_index: input_block_index,
- } = input_block_path;
-
- let input_date_data = states
+ let input_date_data =
+ states
.date_data_vec
.get_mut(input_date_index as usize)
.unwrap_or_else(|| {
@@ -430,121 +398,112 @@ pub fn parse(
panic!()
});
- let input_block_data = input_date_data
- .blocks
- .get_mut(input_block_index as usize)
+ let input_block_data = input_date_data
+ .blocks
+ .get_mut(input_block_index as usize)
+ .unwrap_or_else(|| {
+ dbg!(
+ height,
+ &input_txid,
+ input_block_path,
+ input_date_index,
+ input_block_index,
+ );
+ panic!()
+ });
+
+ input_block_data.send(input_amount);
+
+ inputs_sum += input_amount;
+
+ block_path_to_sent_data
+ .entry(input_block_path)
+ .or_default()
+ .send(input_amount);
+
+ satblocks_destroyed += input_amount * (height - input_block_data.height);
+
+ satdays_destroyed +=
+ input_amount * date.signed_duration_since(*input_date_data.date).num_days() as u64;
+
+ if compute_addresses {
+ let input_address_index = input_address_index.unwrap_or_else(|| {
+ dbg!(
+ height,
+ input_amount,
+ &input_tx_data,
+ input_address_index,
+ input_txout_index,
+ txid,
+ input_txid,
+ input_vout
+ );
+ panic!()
+ });
+
+ let address_index_to_address_data = address_index_to_address_data.as_mut().unwrap();
+
+ let input_address_data = address_index_to_address_data
+ .get_mut(&input_address_index)
.unwrap_or_else(|| {
- dbg!(
- height,
- &input_txid,
- input_block_path,
- input_date_index,
- input_block_index,
- );
- panic!()
+ dbg!(input_address_index, input_txout_index, input_txid, input_vout);
+ panic!();
});
- input_block_data.send(input_amount);
+ let input_address_realized_data = address_index_to_address_realized_data
+ .entry(input_address_index)
+ .or_insert_with(|| AddressRealizedData::default(input_address_data));
- inputs_sum += input_amount;
+ let previous_price = input_block_data.price;
- block_path_to_sent_data
- .entry(input_block_path)
- .or_default()
- .send(input_amount);
-
- satblocks_destroyed += input_amount * (height - input_block_data.height);
-
- satdays_destroyed += input_amount
- * date.signed_duration_since(*input_date_data.date).num_days() as u64;
-
- if compute_addresses {
- let input_address_index = input_address_index.unwrap_or_else(|| {
+ // MUST be after `or_insert_with`
+ input_address_data
+ .send(input_amount, previous_price)
+ .unwrap_or_else(|_| {
dbg!(
- height,
- input_amount,
- &input_tx_data,
input_address_index,
- input_txout_index,
txid,
input_txid,
- input_vout
+ input_amount,
+ tx_index,
+ input_tx_index,
+ input_vout,
+ &input_address_data
);
+
panic!()
});
- let address_index_to_address_data =
- address_index_to_address_data.as_mut().unwrap();
-
- let input_address_data = address_index_to_address_data
- .get_mut(&input_address_index)
- .unwrap_or_else(|| {
- dbg!(
- input_address_index,
- input_txout_index,
- input_txid,
- input_vout
- );
- panic!();
- });
-
- let input_address_realized_data =
- address_index_to_address_realized_data
- .entry(input_address_index)
- .or_insert_with(|| {
- AddressRealizedData::default(input_address_data)
- });
-
- let previous_price = input_block_data.price;
-
- // MUST be after `or_insert_with`
- input_address_data
- .send(input_amount, previous_price)
- .unwrap_or_else(|_| {
- dbg!(
- input_address_index,
- txid,
- input_txid,
- input_amount,
- tx_index,
- input_tx_index,
- input_vout,
- &input_address_data
- );
-
- panic!()
- });
-
- input_address_realized_data.send(
- input_amount,
- block_price,
- previous_price,
- timestamp,
- input_block_data.timestamp,
- );
- };
-
- is_tx_data_from_cached_puts && input_tx_data.is_empty()
+ input_address_realized_data.send(
+ input_amount,
+ block_price,
+ previous_price,
+ timestamp,
+ input_block_data.timestamp,
+ );
};
- if remove_tx_data_from_cached_puts {
- // Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
- databases.txid_to_tx_data.remove_from_puts(&input_txid)
- }
+ is_tx_data_from_cached_puts && input_tx_data.is_empty()
+ };
- ControlFlow::Continue(())
- })?;
- }
+ if remove_tx_data_from_cached_puts {
+ // Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
+ databases.txid_to_tx_data.remove_from_ram(&input_txid)
+ }
- amount_sent += inputs_sum;
+ ControlFlow::Continue(())
+ })?;
+ }
- let fee = inputs_sum - outputs_sum;
+ amount_sent += inputs_sum;
- fees_total += fee;
- fees.push(fee);
+ let fee = inputs_sum - outputs_sum;
- ControlFlow::Continue(())
- });
+ fees_total += fee;
+ fees.push(fee);
+
+ ControlFlow::Continue(())
+ });
if !partial_txout_data_vec.is_empty() {
panic!("partial_txout_data_vec should've been fully consumed");
@@ -553,7 +512,7 @@ pub fn parse(
txid_to_tx_data.into_iter().for_each(|(txid, tx_data)| {
if let Some(tx_data) = tx_data {
if tx_data.is_empty() {
- databases.txid_to_tx_data.remove_from_db(txid);
+ databases.txid_to_tx_data.remove_later_from_disk(txid);
} else {
databases.txid_to_tx_data.update(txid, tx_data);
}
@@ -577,24 +536,17 @@ pub fn parse(
if datasets.utxo.needs_durable_states(height, date) {
if let Some(previous_last_block_data) = previous_last_block_data {
- block_path_to_sent_data
- .iter()
- .for_each(|(block_path, sent_data)| {
- let block_data =
- states.date_data_vec.get_block_data(block_path).unwrap();
+ block_path_to_sent_data.iter().for_each(|(block_path, sent_data)| {
+ let block_data = states.date_data_vec.get_block_data(block_path).unwrap();
- if block_data.height != height {
- states
- .utxo_cohorts_durable_states
- .as_mut()
- .unwrap()
- .subtract_moved(
- block_data,
- sent_data,
- previous_last_block_data,
- );
- }
- });
+ if block_data.height != height {
+ states.utxo_cohorts_durable_states.as_mut().unwrap().subtract_moved(
+ block_data,
+ sent_data,
+ previous_last_block_data,
+ );
+ }
+ });
}
let last_block_data = states.date_data_vec.last_block().unwrap();
@@ -612,11 +564,7 @@ pub fn parse(
.utxo_cohorts_durable_states
.as_mut()
.unwrap()
- .udpate_age_if_needed(
- block_data,
- last_block_data,
- previous_last_block_data,
- );
+ .udpate_age_if_needed(block_data, last_block_data, previous_last_block_data);
});
}
@@ -625,14 +573,7 @@ pub fn parse(
.utxo_cohorts_durable_states
.as_ref()
.unwrap()
- .compute_one_shot_states(
- block_price,
- if is_date_last_block {
- Some(date_price)
- } else {
- None
- },
- );
+ .compute_one_shot_states(block_price, if is_date_last_block { Some(date_price) } else { None });
}
});
@@ -665,10 +606,10 @@ pub fn parse(
// TODO: Only compute if needed
address_cohorts_output_states.replace(AddressCohortsOutputStates::default());
- address_index_to_address_realized_data.iter().for_each(
- |(address_index, address_realized_data)| {
- let current_address_data =
- address_index_to_address_data.get(address_index).unwrap();
+ address_index_to_address_realized_data
+ .iter()
+ .for_each(|(address_index, address_realized_data)| {
+ let current_address_data = address_index_to_address_data.get(address_index).unwrap();
states
.address_cohorts_durable_states
@@ -708,42 +649,34 @@ pub fn parse(
¤t_address_data.compute_liquidity_classification(),
)
.unwrap();
- },
- );
+ });
address_cohorts_one_shot_states.replace(
states
.address_cohorts_durable_states
.as_ref()
.unwrap()
- .compute_one_shot_states(
- block_price,
- if is_date_last_block {
- Some(date_price)
- } else {
- None
- },
- ),
+ .compute_one_shot_states(block_price, if is_date_last_block { Some(date_price) } else { None }),
);
});
}
});
if compute_addresses {
- address_index_to_address_data.unwrap().into_iter().for_each(
- |(address_index, address_data)| {
+ address_index_to_address_data
+ .unwrap()
+ .into_iter()
+ .for_each(|(address_index, address_data)| {
if address_data.is_empty() {
- databases.address_index_to_empty_address_data.unsafe_insert(
- address_index,
- EmptyAddressData::from_non_empty(&address_data),
- );
+ databases
+ .address_index_to_empty_address_data
+ .insert_to_ram(address_index, EmptyAddressData::from_non_empty(&address_data));
} else {
databases
.address_index_to_address_data
- .unsafe_insert(address_index, address_data);
+ .insert_to_ram(address_index, address_data);
}
- },
- )
+ })
}
datasets.insert(InsertData {
@@ -848,17 +781,15 @@ fn prepare_outputs(
.collect_vec();
if compute_addresses {
- partial_txout_data_vec
- .par_iter_mut()
- .for_each(|partial_tx_out_data| {
- if let Some(partial_tx_out_data) = partial_tx_out_data {
- let address_index_opt = address_to_address_index
- .unsafe_get(partial_tx_out_data.address.as_ref().unwrap())
- .cloned();
+ partial_txout_data_vec.par_iter_mut().for_each(|partial_tx_out_data| {
+ if let Some(partial_tx_out_data) = partial_tx_out_data {
+ let address_index_opt = address_to_address_index
+ .unsafe_get(partial_tx_out_data.address.as_ref().unwrap())
+ .cloned();
- partial_tx_out_data.address_index_opt = address_index_opt;
- }
- });
+ partial_tx_out_data.address_index_opt = address_index_opt;
+ }
+ });
}
TxoutsParsingResults {
@@ -896,7 +827,7 @@ fn prepare_inputs<'a>(
let mut tx_datas = txid_to_tx_data
.par_iter()
- .map(|(txid, _)| txid_to_tx_data_db.unsafe_get(txid))
+ .map(|(txid, _)| txid_to_tx_data_db.get(txid))
.collect::>();
txid_to_tx_data.values_mut().rev().for_each(|tx_data_opt| {
@@ -934,11 +865,8 @@ fn prepare_inputs<'a>(
// https://mempool.space/tx/9d8a0d851c9fb2cdf1c6d9406ce97e19e6911ae3503ab2dd5f38640bacdac996
// which is used later as input
.map(|amount| {
- let address_index = compute_addresses.then(|| {
- *txout_index_to_address_index_db
- .unsafe_get(&txout_index)
- .unwrap()
- });
+ let address_index =
+ compute_addresses.then(|| *txout_index_to_address_index_db.unsafe_get(&txout_index).unwrap());
(txout_index, (*amount, address_index))
})
@@ -986,21 +914,16 @@ fn compute_address_index_to_address_data(
address_index_to_address_data
.par_iter_mut()
.for_each(|(address_index, address_data)| {
- if let Some(_address_data) =
- address_index_to_address_data_db.unsafe_get_from_cache(address_index)
- {
+ if let Some(_address_data) = address_index_to_address_data_db.get_from_ram(address_index) {
_address_data.clone_into(address_data);
- } else if let Some(empty_address_data) =
- address_index_to_empty_address_data_db.unsafe_get_from_cache(address_index)
+ } else if let Some(empty_address_data) = address_index_to_empty_address_data_db.get_from_ram(address_index)
{
*address_data = AddressData::from_empty(empty_address_data);
- } else if let Some(_address_data) =
- address_index_to_address_data_db.unsafe_get_from_db(address_index)
- {
+ } else if let Some(_address_data) = address_index_to_address_data_db.get_from_disk(address_index) {
_address_data.clone_into(address_data);
} else {
let empty_address_data = address_index_to_empty_address_data_db
- .unsafe_get_from_db(address_index)
+ .get_from_disk(address_index)
.unwrap();
*address_data = AddressData::from_empty(empty_address_data);
diff --git a/_src/parser/databases/_trait.rs b/_src/parser/databases/_trait.rs
new file mode 100644
index 000000000..5da93dd8a
--- /dev/null
+++ b/_src/parser/databases/_trait.rs
@@ -0,0 +1,60 @@
+use std::{fs, io, path::Path};
+
+use log::info;
+use snkrj::AnyDatabase;
+
+use crate::structs::{Config, Date, Height};
+
+use super::Metadata;
+
+pub trait AnyDatabaseGroup
+where
+ Self: Sized,
+{
+ fn init(config: &Config) -> Self {
+ let s = Self::import(config);
+ s.create_dir_all().unwrap();
+ s
+ }
+
+ fn import(config: &Config) -> Self;
+
+ fn drain_to_vec(&mut self) -> Vec>;
+ fn open_all(&mut self);
+
+ fn metadata(&mut self) -> &mut Metadata;
+ fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
+ self.metadata().export(height, date)
+ }
+
+ fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
+ fs::create_dir_all(self.path())
+ }
+
+ fn remove_dir_all(&self) -> color_eyre::Result<(), io::Error> {
+ fs::remove_dir_all(self.path())
+ }
+
+ fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
+ info!(
+ "Reset {}",
+ self.path()
+ .components()
+ .last()
+ .unwrap()
+ .as_os_str()
+ .to_str()
+ .unwrap()
+ );
+
+ self.reset_metadata();
+ self.remove_dir_all()?;
+ self.create_dir_all()?;
+
+ Ok(())
+ }
+
+ fn reset_metadata(&mut self);
+
+ fn path(&self) -> &Path;
+}
diff --git a/parser/src/databases/address_index_to_address_data.rs b/_src/parser/databases/address_index_to_address_data.rs
similarity index 63%
rename from parser/src/databases/address_index_to_address_data.rs
rename to _src/parser/databases/address_index_to_address_data.rs
index e6ed7fc80..fdcb09205 100644
--- a/parser/src/databases/address_index_to_address_data.rs
+++ b/_src/parser/databases/address_index_to_address_data.rs
@@ -2,19 +2,20 @@ use std::{
collections::BTreeMap,
fs, mem,
ops::{Deref, DerefMut},
+ path::{Path, PathBuf},
};
use allocative::Allocative;
use itertools::Itertools;
use rayon::prelude::*;
+use snkrj::{AnyDatabase, Database as _Database};
use crate::{
- states::AddressCohortsDurableStates,
- structs::{AddressData, Date, Height},
- utils::time,
+ parser::states::AddressCohortsDurableStates,
+ structs::{AddressData, Config},
};
-use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
+use super::{AnyDatabaseGroup, Metadata};
type Key = u32;
type Value = AddressData;
@@ -22,8 +23,9 @@ type Database = _Database;
#[derive(Allocative)]
pub struct AddressIndexToAddressData {
+ path: PathBuf,
pub metadata: Metadata,
-
+ #[allocative(skip)]
pub map: BTreeMap,
}
@@ -44,10 +46,10 @@ impl DerefMut for AddressIndexToAddressData {
pub const ADDRESS_INDEX_DB_MAX_SIZE: usize = 250_000;
impl AddressIndexToAddressData {
- pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option {
+ pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option {
self.metadata.called_insert();
- self.open_db(&key).unsafe_insert(key, value)
+ self.open_db(&key).insert_to_ram(key, value)
}
pub fn remove(&mut self, key: &Key) -> Option {
@@ -58,20 +60,21 @@ impl AddressIndexToAddressData {
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
/// Though it makes it easy to use with rayon.
- pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
+ pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
- self.get(&db_index).unwrap().get_from_puts(key)
+ self.get(&db_index).unwrap().get_from_ram(key)
}
- pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
+ pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
- self.get(&db_index).unwrap().db_get(key)
+ self.get(&db_index).unwrap().get_from_disk(key)
}
pub fn open_db(&mut self, key: &Key) -> &mut Database {
let db_index = Self::db_index(key);
+ let path = self.path().to_owned();
self.entry(db_index).or_insert_with(|| {
let db_name = format!(
@@ -80,31 +83,31 @@ impl AddressIndexToAddressData {
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
);
- let path = Self::root().join(db_name);
+ let path = path.join(db_name);
Database::open(path).unwrap()
})
}
pub fn compute_addres_cohorts_durable_states(&mut self) -> AddressCohortsDurableStates {
- time("Iter through address_index_to_address_data", || {
- self.open_all();
+ // time("Iter through address_index_to_address_data", || {
+ self.open_all();
- // MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
- mem::take(&mut self.map)
- .par_iter()
- .map(|(_, database)| {
- let mut s = AddressCohortsDurableStates::default();
+ // MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
+ mem::take(&mut self.map)
+ .par_iter()
+ .map(|(_, database)| {
+ let mut s = AddressCohortsDurableStates::default();
- database
- .iter()
- .map(|r| r.unwrap().1)
- .for_each(|address_data| s.increment(address_data).unwrap());
+ database
+ .iter_disk()
+ .map(|r| r.unwrap().1)
+ .for_each(|address_data| s.increment(address_data).unwrap());
- s
- })
- .sum()
- })
+ s
+ })
+ .sum()
+ // })
}
fn db_index(key: &Key) -> usize {
@@ -113,30 +116,23 @@ impl AddressIndexToAddressData {
}
impl AnyDatabaseGroup for AddressIndexToAddressData {
- fn import() -> Self {
+ fn import(config: &Config) -> Self {
+ let path = config
+ .path_databases()
+ .join("address_index_to_address_data");
Self {
- metadata: Metadata::import(Self::root(), 1),
-
+ metadata: Metadata::import(&path, 1),
+ path,
map: BTreeMap::default(),
}
}
- fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
- fs::create_dir_all(Self::root())
- }
-
fn reset_metadata(&mut self) {
self.metadata.reset();
}
- fn folder<'a>() -> &'a str {
- "address_index_to_address_data"
- }
-
fn open_all(&mut self) {
- let path = Self::root();
-
- let folder = fs::read_dir(path);
+ let folder = fs::read_dir(&self.path);
if folder.is_err() {
return;
@@ -167,7 +163,11 @@ impl AnyDatabaseGroup for AddressIndexToAddressData {
.collect_vec()
}
- fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
- self.metadata.export(height, date)
+ fn metadata(&mut self) -> &mut Metadata {
+ &mut self.metadata
+ }
+
+ fn path(&self) -> &Path {
+ &self.path
}
}
diff --git a/parser/src/databases/address_index_to_empty_address_data.rs b/_src/parser/databases/address_index_to_empty_address_data.rs
similarity index 73%
rename from parser/src/databases/address_index_to_empty_address_data.rs
rename to _src/parser/databases/address_index_to_empty_address_data.rs
index cdb0127fc..df940c2f3 100644
--- a/parser/src/databases/address_index_to_empty_address_data.rs
+++ b/_src/parser/databases/address_index_to_empty_address_data.rs
@@ -2,16 +2,16 @@ use std::{
collections::BTreeMap,
fs, mem,
ops::{Deref, DerefMut},
+ path::{Path, PathBuf},
};
use allocative::Allocative;
use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
-use crate::structs::{Date, EmptyAddressData, Height};
+use crate::structs::{Config, EmptyAddressData};
-use super::{
- AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE,
-};
+use super::{AnyDatabaseGroup, Metadata, ADDRESS_INDEX_DB_MAX_SIZE};
type Key = u32;
type Value = EmptyAddressData;
@@ -19,8 +19,9 @@ type Database = _Database;
#[derive(Allocative)]
pub struct AddressIndexToEmptyAddressData {
+ path: PathBuf,
pub metadata: Metadata,
-
+ #[allocative(skip)]
map: BTreeMap,
}
@@ -39,10 +40,10 @@ impl DerefMut for AddressIndexToEmptyAddressData {
}
impl AddressIndexToEmptyAddressData {
- pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option {
+ pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option {
self.metadata.called_insert();
- self.open_db(&key).unsafe_insert(key, value)
+ self.open_db(&key).insert_to_ram(key, value)
}
pub fn remove(&mut self, key: &Key) -> Option {
@@ -53,13 +54,13 @@ impl AddressIndexToEmptyAddressData {
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
/// Though it makes it easy to use with rayon.
- pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
+ pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
- self.get(&db_index).and_then(|db| db.get_from_puts(key))
+ self.get(&db_index).and_then(|db| db.get_from_ram(key))
}
- pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
+ pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
let db_index = Self::db_index(key);
self.get(&db_index)
@@ -67,11 +68,12 @@ impl AddressIndexToEmptyAddressData {
dbg!(&self.map.keys(), &key, &db_index);
panic!()
})
- .db_get(key)
+ .get_from_disk(key)
}
pub fn open_db(&mut self, key: &Key) -> &mut Database {
let db_index = Self::db_index(key);
+ let path = self.path.to_owned();
self.entry(db_index).or_insert_with(|| {
let db_name = format!(
@@ -80,7 +82,7 @@ impl AddressIndexToEmptyAddressData {
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
);
- let path = Self::root().join(db_name);
+ let path = path.join(db_name);
Database::open(path).unwrap()
})
@@ -92,30 +94,23 @@ impl AddressIndexToEmptyAddressData {
}
impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
- fn import() -> Self {
+ fn import(config: &Config) -> Self {
+ let path = config
+ .path_databases()
+ .join("address_index_to_empty_address_data");
Self {
- metadata: Metadata::import(Self::root(), 1),
-
+ metadata: Metadata::import(&path, 1),
+ path,
map: BTreeMap::default(),
}
}
- fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
- fs::create_dir_all(Self::root())
- }
-
fn reset_metadata(&mut self) {
self.metadata.reset();
}
- fn folder<'a>() -> &'a str {
- "address_index_to_empty_address_data"
- }
-
fn open_all(&mut self) {
- let path = Self::root();
-
- let folder = fs::read_dir(path);
+ let folder = fs::read_dir(&self.path);
if folder.is_err() {
return;
@@ -146,7 +141,11 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
.collect_vec()
}
- fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
- self.metadata.export(height, date)
+ fn metadata(&mut self) -> &mut Metadata {
+ &mut self.metadata
+ }
+
+ fn path(&self) -> &Path {
+ &self.path
}
}
diff --git a/parser/src/databases/address_to_address_index.rs b/_src/parser/databases/address_to_address_index.rs
similarity index 81%
rename from parser/src/databases/address_to_address_index.rs
rename to _src/parser/databases/address_to_address_index.rs
index 34b7e8d5b..01518a9fe 100644
--- a/parser/src/databases/address_to_address_index.rs
+++ b/_src/parser/databases/address_to_address_index.rs
@@ -6,10 +6,11 @@ use std::{
use allocative::Allocative;
use itertools::Itertools;
+use snkrj::{AnyDatabase, Database};
-use crate::structs::{Address, Date, Height, U8x19, U8x31};
+use crate::structs::{Address, Config, U8x19, U8x31};
-use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata};
+use super::{AnyDatabaseGroup, Metadata};
type Value = u32;
type U8x19Database = Database;
@@ -30,18 +31,30 @@ type MultisigDatabase = U32Database;
#[derive(Allocative)]
pub struct AddressToAddressIndex {
+ path: PathBuf,
pub metadata: Metadata,
+ #[allocative(skip)]
p2pk: BTreeMap,
+ #[allocative(skip)]
p2pkh: BTreeMap,
+ #[allocative(skip)]
p2sh: BTreeMap,
+ #[allocative(skip)]
p2wpkh: BTreeMap,
+ #[allocative(skip)]
p2wsh: BTreeMap,
+ #[allocative(skip)]
p2tr: BTreeMap,
+ #[allocative(skip)]
op_return: Option,
+ #[allocative(skip)]
push_only: Option,
+ #[allocative(skip)]
unknown: Option,
+ #[allocative(skip)]
empty: Option,
+ #[allocative(skip)]
multisig: Option,
}
@@ -102,19 +115,19 @@ impl AddressToAddressIndex {
}
}
- pub fn unsafe_get_from_puts(&self, address: &Address) -> Option<&Value> {
+ pub fn get_from_ram(&self, address: &Address) -> Option<&Value> {
match address {
- Address::Empty(key) => self.empty.as_ref().unwrap().get_from_puts(key),
- Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_puts(key),
- Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_puts(key),
- Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_puts(key),
- Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_puts(key),
- Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_puts(key),
- Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_puts(key),
- Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_puts(key),
- Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_puts(key),
- Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_puts(key),
- Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_puts(key),
+ Address::Empty(key) => self.empty.as_ref().unwrap().get_from_ram(key),
+ Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_ram(key),
+ Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_ram(key),
+ Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_ram(key),
+ Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_ram(key),
+ Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_ram(key),
+ Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_ram(key),
+ Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_ram(key),
+ Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_ram(key),
+ Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_ram(key),
+ Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_ram(key),
}
}
@@ -160,12 +173,12 @@ impl AddressToAddressIndex {
.collect_vec()
}
- fn path_p2pk() -> PathBuf {
- Self::root().join("p2pk")
+ fn path_p2pk(&self) -> PathBuf {
+ self.path().join("p2pk")
}
pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase {
- let path = Self::path_p2pk();
+ let path = self.path_p2pk();
self.p2pk.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
Database::open(path).unwrap()
@@ -173,7 +186,7 @@ impl AddressToAddressIndex {
}
fn open_all_p2pk(&mut self) {
- let path = Self::path_p2pk();
+ let path = self.path_p2pk();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
@@ -184,12 +197,12 @@ impl AddressToAddressIndex {
});
}
- fn path_p2pkh() -> PathBuf {
- Self::root().join("p2pkh")
+ fn path_p2pkh(&self) -> PathBuf {
+ self.path().join("p2pkh")
}
pub fn open_p2pkh(&mut self, prefix: u16) -> &mut P2PKHDatabase {
- let path = Self::path_p2pkh();
+ let path = self.path_p2pkh();
self.p2pkh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
@@ -198,7 +211,7 @@ impl AddressToAddressIndex {
}
fn open_all_p2pkh(&mut self) {
- let path = Self::path_p2pkh();
+ let path = self.path_p2pkh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
@@ -209,12 +222,12 @@ impl AddressToAddressIndex {
});
}
- fn path_p2sh() -> PathBuf {
- Self::root().join("p2sh")
+ fn path_p2sh(&self) -> PathBuf {
+ self.path().join("p2sh")
}
pub fn open_p2sh(&mut self, prefix: u16) -> &mut P2SHDatabase {
- let path = Self::path_p2sh();
+ let path = self.path_p2sh();
self.p2sh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
@@ -223,7 +236,7 @@ impl AddressToAddressIndex {
}
fn open_all_p2sh(&mut self) {
- let path = Self::path_p2sh();
+ let path = self.path_p2sh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
@@ -234,12 +247,12 @@ impl AddressToAddressIndex {
});
}
- fn path_p2wpkh() -> PathBuf {
- Self::root().join("p2wpkh")
+ fn path_p2wpkh(&self) -> PathBuf {
+ self.path().join("p2wpkh")
}
pub fn open_p2wpkh(&mut self, prefix: u16) -> &mut P2WPKHDatabase {
- let path = Self::path_p2wpkh();
+ let path = self.path_p2wpkh();
self.p2wpkh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
@@ -248,7 +261,7 @@ impl AddressToAddressIndex {
}
fn open_all_p2wpkh(&mut self) {
- let path = Self::path_p2wpkh();
+ let path = self.path_p2wpkh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
@@ -259,12 +272,12 @@ impl AddressToAddressIndex {
});
}
- fn path_p2wsh() -> PathBuf {
- Self::root().join("p2wsh")
+ fn path_p2wsh(&self) -> PathBuf {
+ self.path().join("p2wsh")
}
pub fn open_p2wsh(&mut self, prefix: u16) -> &mut P2WSHDatabase {
- let path = Self::path_p2wsh();
+ let path = self.path_p2wsh();
self.p2wsh.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
@@ -273,7 +286,7 @@ impl AddressToAddressIndex {
}
fn open_all_p2wsh(&mut self) {
- let path = Self::path_p2wsh();
+ let path = self.path_p2wsh();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
@@ -284,12 +297,12 @@ impl AddressToAddressIndex {
});
}
- fn path_p2tr() -> PathBuf {
- Self::root().join("p2tr")
+ fn path_p2tr(&self) -> PathBuf {
+ self.path().join("p2tr")
}
pub fn open_p2tr(&mut self, prefix: u16) -> &mut P2TRDatabase {
- let path = Self::path_p2tr();
+ let path = self.path_p2tr();
self.p2tr.entry(prefix).or_insert_with(|| {
let path = path.join(prefix.to_string());
@@ -298,7 +311,7 @@ impl AddressToAddressIndex {
}
fn open_all_p2tr(&mut self) {
- let path = Self::path_p2tr();
+ let path = self.path_p2tr();
Self::path_to_group_prefixes(&path)
.into_iter()
.for_each(|prefix| {
@@ -311,34 +324,36 @@ impl AddressToAddressIndex {
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
self.unknown
- .get_or_insert_with(|| Database::open(Self::root().join("unknown")).unwrap())
+ .get_or_insert_with(|| Database::open(self.path.join("unknown")).unwrap())
}
pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
self.op_return
- .get_or_insert_with(|| Database::open(Self::root().join("op_return")).unwrap())
+ .get_or_insert_with(|| Database::open(self.path.join("op_return")).unwrap())
}
pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
self.push_only
- .get_or_insert_with(|| Database::open(Self::root().join("push_only")).unwrap())
+ .get_or_insert_with(|| Database::open(self.path.join("push_only")).unwrap())
}
pub fn open_empty(&mut self) -> &mut UnknownDatabase {
self.empty
- .get_or_insert_with(|| Database::open(Self::root().join("empty")).unwrap())
+ .get_or_insert_with(|| Database::open(self.path.join("empty")).unwrap())
}
pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
self.multisig
- .get_or_insert_with(|| Database::open(Self::root().join("multisig")).unwrap())
+ .get_or_insert_with(|| Database::open(self.path.join("multisig")).unwrap())
}
}
impl AnyDatabaseGroup for AddressToAddressIndex {
- fn import() -> Self {
+ fn import(config: &Config) -> Self {
+ let path = config.path_databases().join("address_to_address_index");
Self {
- metadata: Metadata::import(Self::root(), 1),
+ metadata: Metadata::import(&path, 1),
+ path,
p2pk: BTreeMap::default(),
p2pkh: BTreeMap::default(),
@@ -355,22 +370,18 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
}
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
- fs::create_dir_all(Self::path_p2pk()).unwrap();
- fs::create_dir_all(Self::path_p2pkh()).unwrap();
- fs::create_dir_all(Self::path_p2sh()).unwrap();
- fs::create_dir_all(Self::path_p2wpkh()).unwrap();
- fs::create_dir_all(Self::path_p2wsh()).unwrap();
- fs::create_dir_all(Self::path_p2tr())
+ fs::create_dir_all(self.path_p2pk()).unwrap();
+ fs::create_dir_all(self.path_p2pkh()).unwrap();
+ fs::create_dir_all(self.path_p2sh()).unwrap();
+ fs::create_dir_all(self.path_p2wpkh()).unwrap();
+ fs::create_dir_all(self.path_p2wsh()).unwrap();
+ fs::create_dir_all(self.path_p2tr())
}
fn reset_metadata(&mut self) {
self.metadata.reset()
}
- fn folder<'a>() -> &'a str {
- "address_to_address_index"
- }
-
fn drain_to_vec(&mut self) -> Vec> {
mem::take(&mut self.p2pk)
.into_values()
@@ -424,7 +435,11 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
self.open_all_p2tr();
}
- fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
- self.metadata.export(height, date)
+ fn metadata(&mut self) -> &mut Metadata {
+ &mut self.metadata
+ }
+
+ fn path(&self) -> &Path {
+ &self.path
}
}
diff --git a/parser/src/databases/metadata.rs b/_src/parser/databases/metadata.rs
similarity index 94%
rename from parser/src/databases/metadata.rs
rename to _src/parser/databases/metadata.rs
index eb0046c5d..9d69c4817 100644
--- a/parser/src/databases/metadata.rs
+++ b/_src/parser/databases/metadata.rs
@@ -10,8 +10,8 @@ use std::{
};
use crate::{
+ io::Serialization,
structs::{Counter, Date, Height},
- Serialization,
};
#[derive(Default, Debug, Encode, Decode, Allocative)]
@@ -35,10 +35,10 @@ impl DerefMut for Metadata {
}
impl Metadata {
- pub fn import(path: PathBuf, version: u16) -> Self {
+ pub fn import(path: &Path, version: u16) -> Self {
Self {
- data: MetadataData::import(&path, version),
- path,
+ data: MetadataData::import(path, version),
+ path: path.to_owned(),
}
}
diff --git a/parser/src/databases/mod.rs b/_src/parser/databases/mod.rs
similarity index 72%
rename from parser/src/databases/mod.rs
rename to _src/parser/databases/mod.rs
index 9ff6bc752..413790404 100644
--- a/parser/src/databases/mod.rs
+++ b/_src/parser/databases/mod.rs
@@ -1,8 +1,5 @@
-use std::thread::{self};
-
use allocative::Allocative;
-mod _database;
mod _trait;
mod address_index_to_address_data;
mod address_index_to_empty_address_data;
@@ -12,24 +9,20 @@ mod txid_to_tx_data;
mod txout_index_to_address_index;
mod txout_index_to_amount;
-pub use _database::*;
use _trait::*;
pub use address_index_to_address_data::*;
pub use address_index_to_empty_address_data::*;
pub use address_to_address_index::*;
use itertools::Itertools;
+use log::info;
use metadata::*;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
+use snkrj::AnyDatabase;
pub use txid_to_tx_data::*;
pub use txout_index_to_address_index::*;
pub use txout_index_to_amount::*;
-use crate::{
- log,
- structs::{Date, Height},
- utils::time,
- Exit,
-};
+use crate::structs::{Config, Date, Height};
#[derive(Allocative)]
pub struct Databases {
@@ -42,18 +35,20 @@ pub struct Databases {
}
impl Databases {
- pub fn import() -> Self {
- let address_index_to_address_data = AddressIndexToAddressData::init();
+ pub fn import(config: &Config) -> Self {
+ let address_index_to_address_data = AddressIndexToAddressData::init(config);
- let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init();
+ let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init(config);
- let address_to_address_index = AddressToAddressIndex::init();
+ let address_to_address_index = AddressToAddressIndex::init(config);
- let txid_to_tx_data = TxidToTxData::init();
+ let txid_to_tx_data = TxidToTxData::init(config);
- let txout_index_to_address_index = TxoutIndexToAddressIndex::init();
+ let txout_index_to_address_index = TxoutIndexToAddressIndex::init(config);
- let txout_index_to_amount = TxoutIndexToAmount::init();
+ let txout_index_to_amount = TxoutIndexToAmount::init(config);
+
+ info!("Imported databases");
Self {
address_index_to_address_data,
@@ -91,62 +86,21 @@ impl Databases {
Ok(())
}
- pub fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
+ pub fn export(
+ &mut self,
+ height: Height,
+ date: Date,
+ defragment: bool,
+ ) -> color_eyre::Result<()> {
self.export_metadata(height, date)?;
self.drain_to_vec()
.into_par_iter()
- .try_for_each(AnyDatabase::boxed_export)?;
+ .try_for_each(|s| AnyDatabase::boxed_export(s, defragment))?;
Ok(())
}
- fn open_all(&mut self) {
- thread::scope(|s| {
- s.spawn(|| {
- self.address_index_to_address_data.open_all();
- });
-
- s.spawn(|| {
- self.address_index_to_empty_address_data.open_all();
- });
-
- s.spawn(|| {
- self.address_to_address_index.open_all();
- });
-
- s.spawn(|| {
- self.txid_to_tx_data.open_all();
- });
-
- s.spawn(|| {
- self.txout_index_to_address_index.open_all();
- });
-
- s.spawn(|| {
- self.txout_index_to_amount.open_all();
- });
- });
- }
-
- pub fn defragment(&mut self, exit: &Exit) {
- exit.block();
-
- log("Databases defragmentation");
-
- time("Defragmenting databases", || {
- time("Opened all databases", || self.open_all());
-
- log("Defragmenting...");
-
- self.drain_to_vec()
- .into_par_iter()
- .for_each(AnyDatabase::boxed_defragment);
- });
-
- exit.unblock();
- }
-
pub fn reset(&mut self, include_addresses: bool) {
if include_addresses {
let _ = self.address_index_to_address_data.reset();
@@ -177,8 +131,8 @@ impl Databases {
pub fn check_if_usable(
&self,
- min_initial_last_address_height: Option,
- min_initial_last_address_date: Option,
+ last_address_height: Option,
+ last_address_date: Option,
) -> bool {
let are_tx_databases_in_sync = self
.txout_index_to_amount
@@ -215,8 +169,7 @@ impl Databases {
return false;
}
- // let are_address_datasets_farer_or_in_sync_with_address_databases =
- min_initial_last_address_height >= self.address_to_address_index.metadata.last_height
- && min_initial_last_address_date >= self.address_to_address_index.metadata.last_date
+ last_address_height >= self.address_to_address_index.metadata.last_height
+ && last_address_date >= self.address_to_address_index.metadata.last_date
}
}
diff --git a/parser/src/databases/txid_to_tx_data.rs b/_src/parser/databases/txid_to_tx_data.rs
similarity index 57%
rename from parser/src/databases/txid_to_tx_data.rs
rename to _src/parser/databases/txid_to_tx_data.rs
index 59a706249..0d3581aec 100644
--- a/parser/src/databases/txid_to_tx_data.rs
+++ b/_src/parser/databases/txid_to_tx_data.rs
@@ -1,16 +1,17 @@
use std::{
collections::BTreeMap,
fs, mem,
- ops::{Deref, DerefMut},
+ path::{Path, PathBuf},
};
use allocative::Allocative;
-use biter::bitcoin::Txid;
+use brk_parser::bitcoin::Txid;
use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
-use crate::structs::{Date, Height, TxData, U8x31};
+use crate::structs::{Config, TxData, U8x31};
-use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
+use super::{AnyDatabaseGroup, Metadata};
type Key = U8x31;
type Value = TxData;
@@ -18,25 +19,12 @@ type Database = _Database;
#[derive(Allocative)]
pub struct TxidToTxData {
+ path: PathBuf,
pub metadata: Metadata,
-
+ #[allocative(skip)]
map: BTreeMap,
}
-impl Deref for TxidToTxData {
- type Target = BTreeMap;
-
- fn deref(&self) -> &Self::Target {
- &self.map
- }
-}
-
-impl DerefMut for TxidToTxData {
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.map
- }
-}
-
impl TxidToTxData {
pub fn insert(&mut self, txid: &Txid, tx_index: Value) -> Option {
self.metadata.called_insert();
@@ -46,53 +34,38 @@ impl TxidToTxData {
self.open_db(txid).insert(txid_key, tx_index)
}
- // pub fn safe_get(&mut self, txid: &Txid) -> Option<&Value> {
- // let txid_key = Self::txid_to_key(txid);
- // self.open_db(txid).get(&txid_key)
- // }
-
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed.
/// Though it makes it easy to use with rayon
- pub fn unsafe_get(&self, txid: &Txid) -> Option<&Value> {
+ pub fn get(&self, txid: &Txid) -> Option<&Value> {
let txid_key = Self::txid_to_key(txid);
let db_index = Self::db_index(txid);
- self.get(&db_index).unwrap().get(&txid_key)
+ self.map.get(&db_index).unwrap().get(&txid_key)
}
- // pub fn unsafe_get_from_puts(&self, txid: &Txid) -> Option<&Value> {
- // let txid_key = Self::txid_to_key(txid);
-
- // let db_index = Self::db_index(txid);
-
- // self.get(&db_index).unwrap().get_from_puts(&txid_key)
- // }
-
- pub fn unsafe_get_mut_from_puts(&mut self, txid: &Txid) -> Option<&mut Value> {
+ pub fn get_mut_from_ram(&mut self, txid: &Txid) -> Option<&mut Value> {
let txid_key = Self::txid_to_key(txid);
let db_index = Self::db_index(txid);
- self.get_mut(&db_index)
- .unwrap()
- .get_mut_from_puts(&txid_key)
+ self.map.get_mut(&db_index).unwrap().get_mut_from_ram(&txid_key)
}
- pub fn remove_from_db(&mut self, txid: &Txid) {
+ pub fn remove_later_from_disk(&mut self, txid: &Txid) {
self.metadata.called_remove();
let txid_key = Self::txid_to_key(txid);
- self.open_db(txid).db_remove(&txid_key);
+ self.open_db(txid).remove_later_from_disk(&txid_key);
}
- pub fn remove_from_puts(&mut self, txid: &Txid) {
+ pub fn remove_from_ram(&mut self, txid: &Txid) {
self.metadata.called_remove();
let txid_key = Self::txid_to_key(txid);
- self.open_db(txid).remove_from_puts(&txid_key);
+ self.open_db(txid).remove_from_ram(&txid_key);
}
pub fn update(&mut self, txid: &Txid, tx_data: TxData) {
@@ -108,9 +81,10 @@ impl TxidToTxData {
}
#[inline(always)]
- pub fn _open_db(&mut self, db_index: u16) -> &mut Database {
- self.entry(db_index).or_insert_with(|| {
- let path = Self::root().join(db_index.to_string());
+ fn _open_db(&mut self, db_index: u16) -> &mut Database {
+ let path = self.path.to_owned();
+ self.map.entry(db_index).or_insert_with(|| {
+ let path = path.join(db_index.to_string());
Database::open(path).unwrap()
})
}
@@ -125,31 +99,23 @@ impl TxidToTxData {
}
impl AnyDatabaseGroup for TxidToTxData {
- fn import() -> Self {
- let metadata = Metadata::import(Self::root(), 2);
+ fn import(config: &Config) -> Self {
+ let path = config.path_databases().join("txid_to_tx_data");
+ let metadata = Metadata::import(&path, 2);
Self {
+ path,
metadata,
map: BTreeMap::default(),
}
}
- fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
- fs::create_dir_all(Self::root())
- }
-
fn reset_metadata(&mut self) {
self.metadata.reset();
}
- fn folder<'a>() -> &'a str {
- "txid_to_tx_data"
- }
-
fn open_all(&mut self) {
- let path = Self::root();
-
- let folder = fs::read_dir(path);
+ let folder = fs::read_dir(&self.path);
if folder.is_err() {
return;
@@ -180,7 +146,11 @@ impl AnyDatabaseGroup for TxidToTxData {
.collect_vec()
}
- fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
- self.metadata.export(height, date)
+ fn metadata(&mut self) -> &mut Metadata {
+ &mut self.metadata
+ }
+
+ fn path(&self) -> &Path {
+ &self.path
}
}
diff --git a/parser/src/databases/txout_index_to_address_index.rs b/_src/parser/databases/txout_index_to_address_index.rs
similarity index 74%
rename from parser/src/databases/txout_index_to_address_index.rs
rename to _src/parser/databases/txout_index_to_address_index.rs
index 79baeac10..4e060ddfc 100644
--- a/parser/src/databases/txout_index_to_address_index.rs
+++ b/_src/parser/databases/txout_index_to_address_index.rs
@@ -2,14 +2,16 @@ use std::{
collections::BTreeMap,
fs, mem,
ops::{Deref, DerefMut},
+ path::{Path, PathBuf},
};
use allocative::Allocative;
use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
-use crate::structs::{Date, Height, TxoutIndex};
+use crate::structs::{Config, TxoutIndex};
-use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
+use super::{AnyDatabaseGroup, Metadata};
type Key = TxoutIndex;
type Value = u32;
@@ -17,8 +19,9 @@ type Database = _Database;
#[derive(Allocative)]
pub struct TxoutIndexToAddressIndex {
+ path: PathBuf,
pub metadata: Metadata,
-
+ #[allocative(skip)]
map: BTreeMap,
}
@@ -39,20 +42,12 @@ impl DerefMut for TxoutIndexToAddressIndex {
const DB_MAX_SIZE: usize = 10_000_000_000;
impl TxoutIndexToAddressIndex {
- pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option {
+ pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option {
self.metadata.called_insert();
- self.open_db(&key).unsafe_insert(key, value)
+ self.open_db(&key).insert_to_ram(key, value)
}
- // pub fn undo_insert(&mut self, key: &Key) -> Option {
- // self.open_db(key).remove_from_puts(key).map(|v| {
- // self.metadata.called_remove();
-
- // v
- // })
- // }
-
pub fn remove(&mut self, key: &Key) -> Option {
self.metadata.called_remove();
@@ -69,6 +64,7 @@ impl TxoutIndexToAddressIndex {
pub fn open_db(&mut self, key: &Key) -> &mut Database {
let db_index = Self::db_index(key);
+ let path = self.path.to_owned();
self.entry(db_index).or_insert_with(|| {
let db_name = format!(
@@ -77,7 +73,7 @@ impl TxoutIndexToAddressIndex {
(db_index + 1) * DB_MAX_SIZE
);
- let path = Self::root().join(db_name);
+ let path = path.join(db_name);
Database::open(path).unwrap()
})
@@ -89,30 +85,21 @@ impl TxoutIndexToAddressIndex {
}
impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
- fn import() -> Self {
+ fn import(config: &Config) -> Self {
+ let path = config.path_databases().join("txout_index_to_address_index");
Self {
- metadata: Metadata::import(Self::root(), 1),
-
+ metadata: Metadata::import(&path, 1),
+ path,
map: BTreeMap::default(),
}
}
- fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
- fs::create_dir_all(Self::root())
- }
-
fn reset_metadata(&mut self) {
self.metadata.reset();
}
- fn folder<'a>() -> &'a str {
- "txout_index_to_address_index"
- }
-
fn open_all(&mut self) {
- let path = Self::root();
-
- let folder = fs::read_dir(path);
+ let folder = fs::read_dir(&self.path);
if folder.is_err() {
return;
@@ -151,7 +138,11 @@ impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
.collect_vec()
}
- fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
- self.metadata.export(height, date)
+ fn metadata(&mut self) -> &mut Metadata {
+ &mut self.metadata
+ }
+
+ fn path(&self) -> &Path {
+ &self.path
}
}
diff --git a/parser/src/databases/txout_index_to_amount.rs b/_src/parser/databases/txout_index_to_amount.rs
similarity index 74%
rename from parser/src/databases/txout_index_to_amount.rs
rename to _src/parser/databases/txout_index_to_amount.rs
index 6774dfdb4..bc86a2c2e 100644
--- a/parser/src/databases/txout_index_to_amount.rs
+++ b/_src/parser/databases/txout_index_to_amount.rs
@@ -2,14 +2,16 @@ use std::{
collections::BTreeMap,
fs, mem,
ops::{Deref, DerefMut},
+ path::{Path, PathBuf},
};
use allocative::Allocative;
use itertools::Itertools;
+use snkrj::{AnyDatabase, Database as _Database};
-use crate::structs::{Amount, Date, Height, TxoutIndex};
+use crate::structs::{Amount, Config, TxoutIndex};
-use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
+use super::{AnyDatabaseGroup, Metadata};
type Key = TxoutIndex;
type Value = Amount;
@@ -17,8 +19,9 @@ type Database = _Database;
#[derive(Allocative)]
pub struct TxoutIndexToAmount {
+ path: PathBuf,
pub metadata: Metadata,
-
+ #[allocative(skip)]
map: BTreeMap,
}
@@ -39,20 +42,12 @@ impl DerefMut for TxoutIndexToAmount {
const DB_MAX_SIZE: usize = 10_000_000_000;
impl TxoutIndexToAmount {
- pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option {
+ pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option {
self.metadata.called_insert();
- self.open_db(&key).unsafe_insert(key, value)
+ self.open_db(&key).insert_to_ram(key, value)
}
- // pub fn undo_insert(&mut self, key: &Key) -> Option {
- // self.open_db(key).remove_from_puts(key).map(|v| {
- // self.metadata.called_remove();
-
- // v
- // })
- // }
-
pub fn remove(&mut self, key: &Key) -> Option {
self.metadata.called_remove();
@@ -69,6 +64,7 @@ impl TxoutIndexToAmount {
pub fn open_db(&mut self, key: &Key) -> &mut Database {
let db_index = Self::db_index(key);
+ let path = self.path.to_owned();
self.entry(db_index).or_insert_with(|| {
let db_name = format!(
@@ -77,7 +73,7 @@ impl TxoutIndexToAmount {
(db_index + 1) * DB_MAX_SIZE
);
- let path = Self::root().join(db_name);
+ let path = path.join(db_name);
Database::open(path).unwrap()
})
@@ -89,30 +85,21 @@ impl TxoutIndexToAmount {
}
impl AnyDatabaseGroup for TxoutIndexToAmount {
- fn import() -> Self {
+ fn import(config: &Config) -> Self {
+ let path = config.path_databases().join("txout_index_to_amount");
Self {
- metadata: Metadata::import(Self::root(), 1),
-
+ metadata: Metadata::import(&path, 1),
+ path,
map: BTreeMap::default(),
}
}
- fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
- fs::create_dir_all(Self::root())
- }
-
fn reset_metadata(&mut self) {
self.metadata.reset();
}
- fn folder<'a>() -> &'a str {
- "txout_index_to_amount"
- }
-
fn open_all(&mut self) {
- let path = Self::root();
-
- let folder = fs::read_dir(path);
+ let folder = fs::read_dir(&self.path);
if folder.is_err() {
return;
@@ -151,7 +138,11 @@ impl AnyDatabaseGroup for TxoutIndexToAmount {
.collect_vec()
}
- fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
- self.metadata.export(height, date)
+ fn metadata(&mut self) -> &mut Metadata {
+ &mut self.metadata
+ }
+
+ fn path(&self) -> &Path {
+ &self.path
}
}
diff --git a/parser/src/datasets/_traits/any_dataset.rs b/_src/parser/datasets/_traits/any_dataset.rs
similarity index 99%
rename from parser/src/datasets/_traits/any_dataset.rs
rename to _src/parser/datasets/_traits/any_dataset.rs
index 6a5091efc..4f5eba00b 100644
--- a/parser/src/datasets/_traits/any_dataset.rs
+++ b/_src/parser/datasets/_traits/any_dataset.rs
@@ -3,14 +3,14 @@ use rayon::prelude::*;
use struct_iterable::Iterable;
use crate::{
- datasets::{
+ parser::datasets::{
cohort_metadata::AddressCohortMetadataDataset, ComputeData, DateRecapDataset, RatioDataset,
SubDataset,
},
structs::{
- AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, BiMap, Date, Height, MapKind, Timestamp, OHLC,
+ AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, BiMap, Date, DateMap, Height, HeightMap,
+ MapKind, Timestamp, OHLC,
},
- DateMap, HeightMap,
};
use super::{AnyDatasetGroup, MinInitialStates};
diff --git a/parser/src/datasets/_traits/any_dataset_group.rs b/_src/parser/datasets/_traits/any_dataset_group.rs
similarity index 100%
rename from parser/src/datasets/_traits/any_dataset_group.rs
rename to _src/parser/datasets/_traits/any_dataset_group.rs
diff --git a/parser/src/datasets/_traits/any_datasets.rs b/_src/parser/datasets/_traits/any_datasets.rs
similarity index 100%
rename from parser/src/datasets/_traits/any_datasets.rs
rename to _src/parser/datasets/_traits/any_datasets.rs
diff --git a/parser/src/datasets/_traits/min_initial_state.rs b/_src/parser/datasets/_traits/min_initial_state.rs
similarity index 97%
rename from parser/src/datasets/_traits/min_initial_state.rs
rename to _src/parser/datasets/_traits/min_initial_state.rs
index db747aea8..d89674eb7 100644
--- a/parser/src/datasets/_traits/min_initial_state.rs
+++ b/_src/parser/datasets/_traits/min_initial_state.rs
@@ -29,6 +29,10 @@ impl MinInitialStates {
computed: MinInitialState::compute_from_datasets(datasets, Mode::Computed, config),
}
}
+
+ pub fn min_last_height(&self) -> Option {
+ self.computed.last_height.min(self.inserted.last_height)
+ }
}
#[derive(Default, Debug, Clone, Copy, Allocative)]
@@ -45,13 +49,6 @@ enum Mode {
}
impl MinInitialState {
- // pub fn consume(&mut self, other: Self) {
- // self.first_unsafe_date = other.first_unsafe_date;
- // self.first_unsafe_height = other.first_unsafe_height;
- // self.last_date = other.last_date;
- // self.last_height = other.last_height;
- // }
-
fn compute_from_datasets(datasets: &dyn AnyDatasets, mode: Mode, config: &Config) -> Self {
match mode {
Mode::Inserted => {
diff --git a/parser/src/datasets/_traits/mod.rs b/_src/parser/datasets/_traits/mod.rs
similarity index 100%
rename from parser/src/datasets/_traits/mod.rs
rename to _src/parser/datasets/_traits/mod.rs
diff --git a/parser/src/datasets/address/all_metadata.rs b/_src/parser/datasets/address/all_metadata.rs
similarity index 88%
rename from parser/src/datasets/address/all_metadata.rs
rename to _src/parser/datasets/address/all_metadata.rs
index 124c27e6d..1087d62c6 100644
--- a/parser/src/datasets/address/all_metadata.rs
+++ b/_src/parser/datasets/address/all_metadata.rs
@@ -2,8 +2,8 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
- structs::{BiMap, Config, MapKind},
+ parser::datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
+ structs::{BiMap, Config, MapKind, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -15,8 +15,8 @@ pub struct AllAddressesMetadataDataset {
}
impl AllAddressesMetadataDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(path: &MapPath, config: &Config) -> color_eyre::Result {
+ let f = |s: &str| path.join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/address/cohort.rs b/_src/parser/datasets/address/cohort.rs
similarity index 96%
rename from parser/src/datasets/address/cohort.rs
rename to _src/parser/datasets/address/cohort.rs
index c7d24ac82..f7676ad83 100644
--- a/parser/src/datasets/address/cohort.rs
+++ b/_src/parser/datasets/address/cohort.rs
@@ -3,9 +3,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
- states::{AddressCohortId, DurableStates},
- structs::{AddressSplit, BiMap, Config, Date, Height},
+ parser::{
+ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
+ states::{AddressCohortId, DurableStates},
+ },
+ structs::{AddressSplit, BiMap, Config, Date, Height, MapPath},
};
use super::cohort_metadata::AddressCohortMetadataDataset;
@@ -23,7 +25,7 @@ pub struct CohortDataset {
impl CohortDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
id: AddressCohortId,
config: &Config,
) -> color_eyre::Result {
@@ -33,8 +35,8 @@ impl CohortDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
split,
- metadata: AddressCohortMetadataDataset::import(parent_path, &name, config)?,
- subs: SubDataset::import(parent_path, &name, config)?,
+ metadata: AddressCohortMetadataDataset::import(path, &name, config)?,
+ subs: SubDataset::import(path, &name, config)?,
};
s.min_initial_states
diff --git a/parser/src/datasets/address/cohort_metadata.rs b/_src/parser/datasets/address/cohort_metadata.rs
similarity index 87%
rename from parser/src/datasets/address/cohort_metadata.rs
rename to _src/parser/datasets/address/cohort_metadata.rs
index 185cda2de..860596d0c 100644
--- a/parser/src/datasets/address/cohort_metadata.rs
+++ b/_src/parser/datasets/address/cohort_metadata.rs
@@ -2,8 +2,8 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, InsertData, MinInitialStates},
- structs::{BiMap, Config, MapKind},
+ parser::datasets::{AnyDataset, InsertData, MinInitialStates},
+ structs::{BiMap, Config, MapKind, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -19,15 +19,15 @@ pub struct AddressCohortMetadataDataset {
impl AddressCohortMetadataDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
diff --git a/parser/src/datasets/address/mod.rs b/_src/parser/datasets/address/mod.rs
similarity index 93%
rename from parser/src/datasets/address/mod.rs
rename to _src/parser/datasets/address/mod.rs
index 392bf8bd6..a8760c860 100644
--- a/parser/src/datasets/address/mod.rs
+++ b/_src/parser/datasets/address/mod.rs
@@ -7,9 +7,8 @@ use itertools::Itertools;
use rayon::prelude::*;
use crate::{
- states::SplitByAddressCohort,
- structs::{BiMap, Config, Height},
- Date,
+ parser::states::SplitByAddressCohort,
+ structs::{BiMap, Config, Date, Height},
};
use self::{all_metadata::AllAddressesMetadataDataset, cohort::CohortDataset};
@@ -26,13 +25,15 @@ pub struct AddressDatasets {
}
impl AddressDatasets {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
+ pub fn import(config: &Config) -> color_eyre::Result {
let mut cohorts = SplitByAddressCohort::>::default();
+ let path_dataset = config.path_datasets();
+
cohorts
.as_vec()
.into_par_iter()
- .map(|(_, id)| (id, CohortDataset::import(parent_path, id, config)))
+ .map(|(_, id)| (id, CohortDataset::import(&path_dataset, id, config)))
.collect::>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
@@ -43,7 +44,7 @@ impl AddressDatasets {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
- metadata: AllAddressesMetadataDataset::import(parent_path, config)?,
+ metadata: AllAddressesMetadataDataset::import(&path_dataset, config)?,
cohorts: cohorts.unwrap(),
};
diff --git a/parser/src/datasets/block_metadata.rs b/_src/parser/datasets/block_metadata.rs
similarity index 87%
rename from parser/src/datasets/block_metadata.rs
rename to _src/parser/datasets/block_metadata.rs
index aaa6ddfe9..1bfd17b0a 100644
--- a/parser/src/datasets/block_metadata.rs
+++ b/_src/parser/datasets/block_metadata.rs
@@ -2,7 +2,7 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::AnyDataset,
+ parser::datasets::AnyDataset,
structs::{Config, Date, HeightMap, MapKind, Timestamp},
};
@@ -16,8 +16,8 @@ pub struct BlockMetadataDataset {
}
impl BlockMetadataDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let f = |s: &str| config.path_datasets().join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/coindays.rs b/_src/parser/datasets/coindays.rs
similarity index 87%
rename from parser/src/datasets/coindays.rs
rename to _src/parser/datasets/coindays.rs
index aae63dfc1..c6eca0151 100644
--- a/parser/src/datasets/coindays.rs
+++ b/_src/parser/datasets/coindays.rs
@@ -2,9 +2,8 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::AnyDataset,
- structs::{Config, MapKind},
- DateMap, HeightMap,
+ parser::datasets::AnyDataset,
+ structs::{Config, DateMap, HeightMap, MapKind},
};
use super::{InsertData, MinInitialStates};
@@ -17,8 +16,8 @@ pub struct CoindaysDataset {
}
impl CoindaysDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let f = |s: &str| config.path_datasets().join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/cointime.rs b/_src/parser/datasets/cointime.rs
similarity index 97%
rename from parser/src/datasets/cointime.rs
rename to _src/parser/datasets/cointime.rs
index 0cd6d6773..2c4b79271 100644
--- a/parser/src/datasets/cointime.rs
+++ b/_src/parser/datasets/cointime.rs
@@ -2,9 +2,8 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- structs::{BiMap, Config, DateMap, Height, MapKind},
+ structs::{BiMap, Config, DateMap, Height, HeightMap, MapKind},
utils::{ONE_DAY_IN_DAYS, ONE_YEAR_IN_DAYS, THREE_MONTHS_IN_DAYS, TWO_WEEK_IN_DAYS},
- HeightMap,
};
use super::{AnyDataset, ComputeData, InsertData, MinInitialStates, RatioDataset};
@@ -72,8 +71,9 @@ pub struct CointimeDataset {
}
impl CointimeDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let path_dataset = config.path_datasets();
+ let f = |s: &str| path_dataset.join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
@@ -93,7 +93,7 @@ impl CointimeDataset {
// Computed
active_cap: BiMap::new_bin(1, MapKind::Computed, &f("active_cap")),
active_price: BiMap::new_bin(1, MapKind::Computed, &f("active_price")),
- active_price_ratio: RatioDataset::import(parent_path, "active_price", config)?,
+ active_price_ratio: RatioDataset::import(&path_dataset, "active_price", config)?,
active_supply: BiMap::new_bin(1, MapKind::Computed, &f("active_supply")),
active_supply_3m_net_change: BiMap::new_bin(
1,
@@ -140,7 +140,7 @@ impl CointimeDataset {
),
cointime_cap: BiMap::new_bin(1, MapKind::Computed, &f("cointime_cap")),
cointime_price: BiMap::new_bin(1, MapKind::Computed, &f("cointime_price")),
- cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price", config)?,
+ cointime_price_ratio: RatioDataset::import(&path_dataset, "cointime_price", config)?,
cointime_value_created: HeightMap::new_bin(
1,
MapKind::Computed,
@@ -237,7 +237,11 @@ impl CointimeDataset {
&f("true_market_deviation"),
),
true_market_mean: BiMap::new_bin(1, MapKind::Computed, &f("true_market_mean")),
- true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean", config)?,
+ true_market_mean_ratio: RatioDataset::import(
+ &path_dataset,
+ "true_market_mean",
+ config,
+ )?,
true_market_net_unrealized_profit_and_loss: BiMap::new_bin(
1,
MapKind::Computed,
@@ -245,7 +249,7 @@ impl CointimeDataset {
),
vaulted_cap: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_cap")),
vaulted_price: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_price")),
- vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price", config)?,
+ vaulted_price_ratio: RatioDataset::import(&path_dataset, "vaulted_price", config)?,
vaulted_supply: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_supply")),
vaulted_supply_3m_net_change: BiMap::new_bin(
1,
diff --git a/parser/src/datasets/constant.rs b/_src/parser/datasets/constant.rs
similarity index 90%
rename from parser/src/datasets/constant.rs
rename to _src/parser/datasets/constant.rs
index ae94965ae..4f1dce63f 100644
--- a/parser/src/datasets/constant.rs
+++ b/_src/parser/datasets/constant.rs
@@ -16,8 +16,8 @@ pub struct ConstantDataset {
}
impl ConstantDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let f = |s: &str| config.path_datasets().join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/date_metadata.rs b/_src/parser/datasets/date_metadata.rs
similarity index 88%
rename from parser/src/datasets/date_metadata.rs
rename to _src/parser/datasets/date_metadata.rs
index 4f69ec24f..80f8fa9d6 100644
--- a/parser/src/datasets/date_metadata.rs
+++ b/_src/parser/datasets/date_metadata.rs
@@ -2,7 +2,7 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::AnyDataset,
+ parser::datasets::AnyDataset,
structs::{Config, DateMap, Height, MapKind},
};
@@ -17,8 +17,8 @@ pub struct DateMetadataDataset {
}
impl DateMetadataDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let f = |s: &str| config.path_datasets().join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/mining.rs b/_src/parser/datasets/mining.rs
similarity index 99%
rename from parser/src/datasets/mining.rs
rename to _src/parser/datasets/mining.rs
index 759fcf35b..667b0a8a7 100644
--- a/parser/src/datasets/mining.rs
+++ b/_src/parser/datasets/mining.rs
@@ -4,7 +4,7 @@ use ordered_float::OrderedFloat;
use struct_iterable::Iterable;
use crate::{
- datasets::AnyDataset,
+ parser::datasets::AnyDataset,
structs::{Amount, BiMap, Config, DateMap, Height, HeightMap, MapKey, MapKind},
utils::{
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
@@ -123,8 +123,8 @@ pub struct MiningDataset {
}
impl MiningDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let f = |s: &str| config.path_datasets().join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/mod.rs b/_src/parser/datasets/mod.rs
similarity index 66%
rename from parser/src/datasets/mod.rs
rename to _src/parser/datasets/mod.rs
index 43a8c94c1..e8b797b18 100644
--- a/parser/src/datasets/mod.rs
+++ b/_src/parser/datasets/mod.rs
@@ -1,9 +1,10 @@
-use std::{collections::BTreeMap, fs, ops::RangeInclusive, path::Path};
+use std::{collections::BTreeMap, ops::RangeInclusive};
use allocative::Allocative;
use itertools::Itertools;
+use log::info;
use rayon::prelude::*;
mod _traits;
@@ -34,16 +35,18 @@ pub use transaction::*;
pub use utxo::*;
use crate::{
- databases::Databases,
- io::{Json, JSON_EXTENSION},
- states::{
- AddressCohortsInputStates,
- AddressCohortsOneShotStates,
- AddressCohortsRealizedStates,
- States,
- UTXOCohortsOneShotStates,
- // UTXOCohortsReceivedStates,
- UTXOCohortsSentStates,
+ io::Json,
+ parser::{
+ databases::Databases,
+ states::{
+ AddressCohortsInputStates,
+ AddressCohortsOneShotStates,
+ AddressCohortsRealizedStates,
+ States,
+ UTXOCohortsOneShotStates,
+ // UTXOCohortsReceivedStates,
+ UTXOCohortsSentStates,
+ },
},
structs::{Amount, Config, Date, Height, Price, Timestamp},
};
@@ -84,7 +87,7 @@ pub struct ComputeData<'a> {
}
#[derive(Allocative)]
-pub struct AllDatasets {
+pub struct Datasets {
min_initial_states: MinInitialStates,
pub constant: ConstantDataset,
@@ -99,31 +102,27 @@ pub struct AllDatasets {
pub utxo: UTXODatasets,
}
-const DATASETS_PATH: &str = "../datasets";
-
-impl AllDatasets {
+impl Datasets {
pub fn import(config: &Config) -> color_eyre::Result {
- let path = DATASETS_PATH;
+ let price = PriceDatasets::import(config)?;
- let price = PriceDatasets::import(path, config)?;
+ let constant = ConstantDataset::import(config)?;
- let constant = ConstantDataset::import(path, config)?;
+ let date_metadata = DateMetadataDataset::import(config)?;
- let date_metadata = DateMetadataDataset::import(path, config)?;
+ let cointime = CointimeDataset::import(config)?;
- let cointime = CointimeDataset::import(path, config)?;
+ let coindays = CoindaysDataset::import(config)?;
- let coindays = CoindaysDataset::import(path, config)?;
+ let mining = MiningDataset::import(config)?;
- let mining = MiningDataset::import(path, config)?;
+ let block_metadata = BlockMetadataDataset::import(config)?;
- let block_metadata = BlockMetadataDataset::import(path, config)?;
+ let transaction = TransactionDataset::import(config)?;
- let transaction = TransactionDataset::import(path, config)?;
+ let address = AddressDatasets::import(config)?;
- let address = AddressDatasets::import(path, config)?;
-
- let utxo = UTXODatasets::import(path, config)?;
+ let utxo = UTXODatasets::import(config)?;
let mut s = Self {
min_initial_states: MinInitialStates::default(),
@@ -140,14 +139,18 @@ impl AllDatasets {
utxo,
};
- s.min_initial_states
- .consume(MinInitialStates::compute_from_datasets(&s, config));
+ s.set_initial_states(config);
- s.export_meta_files()?;
+ info!("Imported datasets");
Ok(s)
}
+ fn set_initial_states(&mut self, config: &Config) {
+ self.min_initial_states
+ .consume(MinInitialStates::compute_from_datasets(self, config));
+ }
+
pub fn insert(&mut self, insert_data: InsertData) {
if insert_data.compute_addresses {
self.address.insert(&insert_data);
@@ -229,21 +232,6 @@ impl AllDatasets {
&mut self.price.market_cap,
);
- // No compute needed for now
- // if self.block_metadata.should_compute(height, date) {
- // self.block_metadata.compute(&compute_data);
- // }
-
- // No compute needed for now
- // if self.date_metadata.should_compute(height, date) {
- // self.date_metadata.compute(&compute_data);
- // }
-
- // No compute needed for now
- // if self.coindays.should_compute(height, date) {
- // self.coindays.compute(&compute_data);
- // }
-
if self.transaction.should_compute(&compute_data) {
self.transaction.compute(
&compute_data,
@@ -269,43 +257,12 @@ impl AllDatasets {
}
}
- pub fn export_meta_files(&self) -> color_eyre::Result<()> {
- let mut path_to_type: BTreeMap<&Path, &str> = self
- .to_any_dataset_vec()
- .into_iter()
- .flat_map(|dataset| dataset.to_all_map_vec())
- .flat_map(|map| map.exported_path_with_t_name())
- .collect();
+ pub fn export(&mut self, config: &Config, height: Height) -> color_eyre::Result<()> {
+ let is_new = self
+ .min_initial_states
+ .min_last_height()
+ .map_or(true, |last| last <= height);
- path_to_type.insert(Path::new("../datasets/last"), "Value");
-
- let datasets_len = path_to_type.len();
-
- let server_inputs_path = "../server/in";
-
- fs::create_dir_all(server_inputs_path)?;
-
- Json::export(
- Path::new(&format!("{server_inputs_path}/disk_path_to_type.json")),
- &path_to_type,
- )?;
-
- let datasets_len_path = format!("{server_inputs_path}/datasets_len.txt");
-
- if let Ok(len) = fs::read_to_string(&datasets_len_path) {
- if let Ok(len) = len.parse::() {
- if datasets_len == len {
- return Ok(());
- }
- }
- }
-
- fs::write(datasets_len_path, datasets_len.to_string())?;
-
- Ok(())
- }
-
- pub fn export(&mut self) -> color_eyre::Result<()> {
self.to_mut_any_dataset_vec()
.into_iter()
.for_each(|dataset| dataset.pre_export());
@@ -321,39 +278,28 @@ impl AllDatasets {
.for_each(|dataset| {
dataset.post_export();
- dataset.to_all_map_vec().iter().for_each(|map| {
- if let Some(last_path) = map.path_last() {
- if let Some(last_value) = map.last_value() {
- let mut last_path = last_path.clone();
- last_path.pop();
-
- let last_path = last_path.to_str().unwrap();
-
- let skip = if last_path.starts_with(DATASETS_PATH) {
- 2
- } else {
- 1
- };
-
- path_to_last.insert(
- last_path.split('/').skip(skip).join("-").to_string(),
- last_value,
- );
+ if is_new {
+ dataset.to_all_map_vec().iter().for_each(|map| {
+ if map.path_last().is_some() {
+ if let Some(last_value) = map.last_value() {
+ path_to_last.insert(map.id(config), last_value);
+ }
}
- }
- });
+ });
+ }
});
- Json::export(
- Path::new(&format!("{DATASETS_PATH}/last.{JSON_EXTENSION}")),
- &path_to_last,
- )?;
+ if is_new {
+ Json::export(&config.path_datasets_last_values(), &path_to_last)?;
+ }
+
+ self.set_initial_states(config);
Ok(())
}
}
-impl AnyDatasets for AllDatasets {
+impl AnyDatasets for Datasets {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
diff --git a/parser/src/datasets/price.rs b/_src/parser/datasets/price.rs
similarity index 93%
rename from parser/src/datasets/price.rs
rename to _src/parser/datasets/price.rs
index d2735c4ee..ca79bd945 100644
--- a/parser/src/datasets/price.rs
+++ b/_src/parser/datasets/price.rs
@@ -7,7 +7,7 @@ use color_eyre::eyre::Error;
use struct_iterable::Iterable;
use crate::{
- price::{Binance, Kibo, Kraken},
+ parser::price::{Binance, Kibo, Kraken},
structs::{
Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey,
MapKind, Timestamp, OHLC,
@@ -85,10 +85,9 @@ pub struct PriceDatasets {
}
impl PriceDatasets {
- pub fn import(datasets_path: &str, config: &Config) -> color_eyre::Result {
- let price_path = "../price";
-
- let f = |s: &str| format!("{datasets_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let path_dataset = config.path_datasets();
+ let f = |s: &str| path_dataset.join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
@@ -104,7 +103,7 @@ impl PriceDatasets {
// ---
// Inserted
// ---
- ohlc: BiMap::new_json(1, MapKind::Inserted, price_path),
+ ohlc: BiMap::new_json(1, MapKind::Inserted, &config.path_price()),
// ---
// Computed
@@ -115,31 +114,31 @@ impl PriceDatasets {
close: BiMap::new_bin(1, MapKind::Computed, &f("close")),
market_cap: BiMap::new_bin(1, MapKind::Computed, &f("market_cap")),
price_1w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1w_sma")),
- price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma", config)?,
+ price_1w_sma_ratio: RatioDataset::import(&path_dataset, "price_1w_sma", config)?,
price_1m_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1m_sma")),
- price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma", config)?,
+ price_1m_sma_ratio: RatioDataset::import(&path_dataset, "price_1m_sma", config)?,
price_1y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1y_sma")),
- price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma", config)?,
+ price_1y_sma_ratio: RatioDataset::import(&path_dataset, "price_1y_sma", config)?,
price_2y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_2y_sma")),
- price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma", config)?,
+ price_2y_sma_ratio: RatioDataset::import(&path_dataset, "price_2y_sma", config)?,
price_4y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_4y_sma")),
- price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma", config)?,
+ price_4y_sma_ratio: RatioDataset::import(&path_dataset, "price_4y_sma", config)?,
price_8d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_8d_sma")),
- price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma", config)?,
+ price_8d_sma_ratio: RatioDataset::import(&path_dataset, "price_8d_sma", config)?,
price_13d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_13d_sma")),
- price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma", config)?,
+ price_13d_sma_ratio: RatioDataset::import(&path_dataset, "price_13d_sma", config)?,
price_21d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_21d_sma")),
- price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma", config)?,
+ price_21d_sma_ratio: RatioDataset::import(&path_dataset, "price_21d_sma", config)?,
price_34d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_34d_sma")),
- price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma", config)?,
+ price_34d_sma_ratio: RatioDataset::import(&path_dataset, "price_34d_sma", config)?,
price_55d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_55d_sma")),
- price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma", config)?,
+ price_55d_sma_ratio: RatioDataset::import(&path_dataset, "price_55d_sma", config)?,
price_89d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_89d_sma")),
- price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma", config)?,
+ price_89d_sma_ratio: RatioDataset::import(&path_dataset, "price_89d_sma", config)?,
price_144d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_144d_sma")),
- price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma", config)?,
+ price_144d_sma_ratio: RatioDataset::import(&path_dataset, "price_144d_sma", config)?,
price_200w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_200w_sma")),
- price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma", config)?,
+ price_200w_sma_ratio: RatioDataset::import(&path_dataset, "price_200w_sma", config)?,
price_1d_total_return: DateMap::new_bin(
1,
MapKind::Computed,
@@ -540,6 +539,7 @@ impl PriceDatasets {
height: Height,
timestamp: Timestamp,
previous_timestamp: Option,
+ config: &Config,
) -> color_eyre::Result {
if let Some(ohlc) = self.ohlc.height.get_or_import(&height) {
return Ok(ohlc);
@@ -558,7 +558,7 @@ impl PriceDatasets {
.unwrap_or_else(|_| {
self.get_from_1mn_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
- self.get_from_har_binance(timestamp, previous_timestamp)
+ self.get_from_har_binance(timestamp, previous_timestamp, config)
.unwrap_or_else(|_| {
self.get_from_height_kibo(&height).unwrap_or_else(|_| {
let date = timestamp.to_date();
@@ -659,10 +659,11 @@ How to fix this:
&mut self,
timestamp: Timestamp,
previous_timestamp: Option,
+ config: &Config,
) -> color_eyre::Result {
if self.binance_har.is_none() {
self.binance_har
- .replace(Binance::read_har_file().unwrap_or_default());
+ .replace(Binance::read_har_file(config).unwrap_or_default());
}
Self::find_height_ohlc(
diff --git a/parser/src/datasets/subs/capitalization.rs b/_src/parser/datasets/subs/capitalization.rs
similarity index 90%
rename from parser/src/datasets/subs/capitalization.rs
rename to _src/parser/datasets/subs/capitalization.rs
index 20a792662..834f19a3e 100644
--- a/parser/src/datasets/subs/capitalization.rs
+++ b/_src/parser/datasets/subs/capitalization.rs
@@ -2,9 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
- states::CapitalizationState,
- structs::{BiMap, Config, MapKind},
+ parser::{
+ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
+ states::CapitalizationState,
+ },
+ structs::{BiMap, Config, MapKind, MapPath},
utils::ONE_MONTH_IN_DAYS,
};
@@ -22,15 +24,15 @@ pub struct CapitalizationDataset {
impl CapitalizationDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
@@ -52,7 +54,7 @@ impl CapitalizationDataset {
),
realized_price: BiMap::new_bin(1, MapKind::Computed, &f("realized_price")),
realized_price_ratio: RatioDataset::import(
- parent_path,
+ path,
&format!(
"{}realized_price",
name.as_ref().map_or("".to_owned(), |n| format!("{n}-"))
diff --git a/parser/src/datasets/subs/input.rs b/_src/parser/datasets/subs/input.rs
similarity index 87%
rename from parser/src/datasets/subs/input.rs
rename to _src/parser/datasets/subs/input.rs
index b9091205e..f7522cc03 100644
--- a/parser/src/datasets/subs/input.rs
+++ b/_src/parser/datasets/subs/input.rs
@@ -2,10 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, InsertData, MinInitialStates},
- states::InputState,
- structs::{BiMap, Config, MapKind},
- DateMap, HeightMap,
+ parser::{
+ datasets::{AnyDataset, InsertData, MinInitialStates},
+ states::InputState,
+ },
+ structs::{BiMap, Config, DateMap, HeightMap, MapKind, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -22,15 +23,15 @@ pub struct InputSubDataset {
impl InputSubDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
diff --git a/parser/src/datasets/subs/mod.rs b/_src/parser/datasets/subs/mod.rs
similarity index 95%
rename from parser/src/datasets/subs/mod.rs
rename to _src/parser/datasets/subs/mod.rs
index c34be68cd..566cd8388 100644
--- a/parser/src/datasets/subs/mod.rs
+++ b/_src/parser/datasets/subs/mod.rs
@@ -21,7 +21,10 @@ pub use supply::*;
pub use unrealized::*;
pub use utxo::*;
-use crate::{datasets::AnyDataset, structs::Config};
+use crate::{
+ parser::datasets::AnyDataset,
+ structs::{Config, MapPath},
+};
use super::AnyDatasetGroup;
@@ -39,7 +42,7 @@ pub struct SubDataset {
impl SubDataset {
pub fn import(
- parent_path: &str,
+ parent_path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
diff --git a/parser/src/datasets/subs/output.rs b/_src/parser/datasets/subs/output.rs
similarity index 100%
rename from parser/src/datasets/subs/output.rs
rename to _src/parser/datasets/subs/output.rs
diff --git a/parser/src/datasets/subs/price_paid.rs b/_src/parser/datasets/subs/price_paid.rs
similarity index 96%
rename from parser/src/datasets/subs/price_paid.rs
rename to _src/parser/datasets/subs/price_paid.rs
index ca06ef2b5..7f1bba43d 100644
--- a/parser/src/datasets/subs/price_paid.rs
+++ b/_src/parser/datasets/subs/price_paid.rs
@@ -2,9 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, InsertData, MinInitialStates},
- states::PricePaidState,
- structs::{BiMap, Config, Date, Height, MapKind},
+ parser::{
+ datasets::{AnyDataset, InsertData, MinInitialStates},
+ states::PricePaidState,
+ },
+ structs::{BiMap, Config, Date, Height, MapKind, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -34,15 +36,15 @@ pub struct PricePaidSubDataset {
impl PricePaidSubDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
diff --git a/parser/src/datasets/subs/ratio.rs b/_src/parser/datasets/subs/ratio.rs
similarity index 93%
rename from parser/src/datasets/subs/ratio.rs
rename to _src/parser/datasets/subs/ratio.rs
index d7ca03c3e..270f9dde0 100644
--- a/parser/src/datasets/subs/ratio.rs
+++ b/_src/parser/datasets/subs/ratio.rs
@@ -2,8 +2,8 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, MinInitialStates},
- structs::{BiMap, Config, MapKind},
+ parser::datasets::{AnyDataset, ComputeData, MinInitialStates},
+ structs::{BiMap, Config, MapKind, MapPath},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
@@ -31,9 +31,9 @@ pub struct RatioDataset {
}
impl RatioDataset {
- pub fn import(parent_path: &str, name: &str, config: &Config) -> color_eyre::Result {
- let f_ratio = |s: &str| format!("{parent_path}/market_price_to_{name}_{s}");
- let f_price = |s: &str| format!("{parent_path}/{name}_{s}");
+ pub fn import(path: &MapPath, name: &str, config: &Config) -> color_eyre::Result {
+ let f_ratio = |s: &str| path.join(&format!("market_price_to_{name}_{s}"));
+ let f_price = |s: &str| path.join(&format!("{name}_{s}"));
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/subs/realized.rs b/_src/parser/datasets/subs/realized.rs
similarity index 97%
rename from parser/src/datasets/subs/realized.rs
rename to _src/parser/datasets/subs/realized.rs
index 005d4b389..c5c4ae98b 100644
--- a/parser/src/datasets/subs/realized.rs
+++ b/_src/parser/datasets/subs/realized.rs
@@ -2,11 +2,12 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
- states::RealizedState,
- structs::{BiMap, Config, MapKind, Price},
+ parser::{
+ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
+ states::RealizedState,
+ },
+ structs::{BiMap, Config, DateMap, HeightMap, MapKind, MapPath, Price},
utils::ONE_MONTH_IN_DAYS,
- DateMap, HeightMap,
};
#[derive(Allocative, Iterable)]
@@ -45,18 +46,17 @@ pub struct RealizedSubDataset {
impl RealizedSubDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
-
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/subs/recap.rs b/_src/parser/datasets/subs/recap.rs
similarity index 95%
rename from parser/src/datasets/subs/recap.rs
rename to _src/parser/datasets/subs/recap.rs
index e5661fa53..7dac8d9d3 100644
--- a/parser/src/datasets/subs/recap.rs
+++ b/_src/parser/datasets/subs/recap.rs
@@ -3,12 +3,14 @@ use std::{iter::Sum, ops::Add};
use allocative::Allocative;
use crate::{
- structs::{DateMapChunkId, GenericMap, MapKey, MapKind, MapSerialized, MapValue},
+ structs::{
+ Date, DateMapChunkId, GenericMap, MapChunkId, MapKey, MapKind, MapPath, MapSerialized,
+ MapValue, SerializedDateMap,
+ },
utils::{get_percentile, LossyFrom},
- Date, MapChunkId, SerializedBTreeMap,
};
-pub type DateRecapDataset = RecapDataset>;
+pub type DateRecapDataset = RecapDataset>;
#[derive(Allocative)]
pub struct RecapDataset {
@@ -91,8 +93,8 @@ where
Key: MapKey,
Serialized: MapSerialized,
{
- pub fn import(parent_path: &str, options: RecapOptions) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(path: &MapPath, options: RecapOptions) -> color_eyre::Result {
+ let f = |s: &str| path.join(s);
let s = Self {
// ---
diff --git a/parser/src/datasets/subs/supply.rs b/_src/parser/datasets/subs/supply.rs
similarity index 90%
rename from parser/src/datasets/subs/supply.rs
rename to _src/parser/datasets/subs/supply.rs
index 316ae0904..7fe41efec 100644
--- a/parser/src/datasets/subs/supply.rs
+++ b/_src/parser/datasets/subs/supply.rs
@@ -2,9 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
- states::SupplyState,
- structs::{BiMap, Config, MapKind},
+ parser::{
+ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
+ states::SupplyState,
+ },
+ structs::{BiMap, Config, MapKind, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -19,15 +21,15 @@ pub struct SupplySubDataset {
impl SupplySubDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
diff --git a/parser/src/datasets/subs/unrealized.rs b/_src/parser/datasets/subs/unrealized.rs
similarity index 95%
rename from parser/src/datasets/subs/unrealized.rs
rename to _src/parser/datasets/subs/unrealized.rs
index 7801cf243..6290f589b 100644
--- a/parser/src/datasets/subs/unrealized.rs
+++ b/_src/parser/datasets/subs/unrealized.rs
@@ -2,9 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
- states::UnrealizedState,
- structs::{BiMap, Config, MapKind},
+ parser::{
+ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
+ states::UnrealizedState,
+ },
+ structs::{BiMap, Config, MapKind, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -26,15 +28,15 @@ pub struct UnrealizedSubDataset {
impl UnrealizedSubDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
diff --git a/parser/src/datasets/subs/utxo.rs b/_src/parser/datasets/subs/utxo.rs
similarity index 83%
rename from parser/src/datasets/subs/utxo.rs
rename to _src/parser/datasets/subs/utxo.rs
index 61ecf9f90..4f5a071ae 100644
--- a/parser/src/datasets/subs/utxo.rs
+++ b/_src/parser/datasets/subs/utxo.rs
@@ -2,9 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, InsertData, MinInitialStates},
- states::UTXOState,
- structs::{BiMap, Config, MapKind},
+ parser::{
+ datasets::{AnyDataset, InsertData, MinInitialStates},
+ states::UTXOState,
+ },
+ structs::{BiMap, Config, MapKind, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -16,15 +18,15 @@ pub struct UTXOSubDataset {
impl UTXOSubDataset {
pub fn import(
- parent_path: &str,
+ path: &MapPath,
name: &Option,
config: &Config,
) -> color_eyre::Result {
let f = |s: &str| {
if let Some(name) = name {
- format!("{parent_path}/{name}/{s}")
+ path.join(&format!("{name}/{s}"))
} else {
- format!("{parent_path}/{s}")
+ path.join(s)
}
};
diff --git a/parser/src/datasets/transaction.rs b/_src/parser/datasets/transaction.rs
similarity index 97%
rename from parser/src/datasets/transaction.rs
rename to _src/parser/datasets/transaction.rs
index 7ab3ae300..94704de63 100644
--- a/parser/src/datasets/transaction.rs
+++ b/_src/parser/datasets/transaction.rs
@@ -2,12 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::InsertData,
- structs::{BiMap, Config, HeightMap, MapKind},
+ parser::datasets::InsertData,
+ structs::{BiMap, Config, DateMap, HeightMap, MapKind},
utils::{
ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, TARGET_BLOCKS_PER_DAY,
},
- DateMap,
};
use super::{AnyDataset, ComputeData, MinInitialStates};
@@ -52,8 +51,8 @@ pub struct TransactionDataset {
}
impl TransactionDataset {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
- let f = |s: &str| format!("{parent_path}/{s}");
+ pub fn import(config: &Config) -> color_eyre::Result {
+ let f = |s: &str| config.path_datasets().join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
diff --git a/parser/src/datasets/utxo/dataset.rs b/_src/parser/datasets/utxo/dataset.rs
similarity index 96%
rename from parser/src/datasets/utxo/dataset.rs
rename to _src/parser/datasets/utxo/dataset.rs
index 96b878ddf..01087dbb5 100644
--- a/parser/src/datasets/utxo/dataset.rs
+++ b/_src/parser/datasets/utxo/dataset.rs
@@ -2,9 +2,11 @@ use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
- datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
- states::UTXOCohortId,
- structs::{BiMap, Config, Date, Height},
+ parser::{
+ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
+ states::UTXOCohortId,
+ },
+ structs::{BiMap, Config, Date, Height, MapPath},
};
#[derive(Allocative, Iterable)]
@@ -18,7 +20,7 @@ pub struct UTXODataset {
impl UTXODataset {
pub fn import(
- parent_path: &str,
+ parent_path: &MapPath,
id: UTXOCohortId,
config: &Config,
) -> color_eyre::Result {
diff --git a/parser/src/datasets/utxo/mod.rs b/_src/parser/datasets/utxo/mod.rs
similarity index 94%
rename from parser/src/datasets/utxo/mod.rs
rename to _src/parser/datasets/utxo/mod.rs
index d9bc811bd..c9d0390b4 100644
--- a/parser/src/datasets/utxo/mod.rs
+++ b/_src/parser/datasets/utxo/mod.rs
@@ -7,8 +7,8 @@ use rayon::prelude::*;
use itertools::Itertools;
use crate::{
- datasets::AnyDatasets,
- states::{SplitByUTXOCohort, UTXOCohortId},
+ parser::datasets::AnyDatasets,
+ parser::states::{SplitByUTXOCohort, UTXOCohortId},
structs::{BiMap, Config, Date, Height},
};
@@ -22,13 +22,15 @@ pub struct UTXODatasets {
}
impl UTXODatasets {
- pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result {
+ pub fn import(config: &Config) -> color_eyre::Result {
let mut cohorts = SplitByUTXOCohort::>::default();
+ let path_dataset = config.path_datasets();
+
cohorts
.as_vec()
.into_par_iter()
- .map(|(_, id)| (id, UTXODataset::import(parent_path, id, config)))
+ .map(|(_, id)| (id, UTXODataset::import(&path_dataset, id, config)))
.collect::>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
diff --git a/_src/parser/mod.rs b/_src/parser/mod.rs
new file mode 100644
index 000000000..57fd87c40
--- /dev/null
+++ b/_src/parser/mod.rs
@@ -0,0 +1,42 @@
+use std::{thread::sleep, time::Duration};
+
+use brk_parser::bitcoincore_rpc::{Client, RpcApi};
+
+mod actions;
+mod databases;
+mod datasets;
+mod price;
+mod states;
+
+pub use actions::*;
+pub use databases::*;
+pub use datasets::*;
+use log::info;
+pub use states::*;
+
+use crate::structs::{Config, Exit};
+
+pub fn main(config: &Config, rpc: &Client, exit: &Exit) -> color_eyre::Result<()> {
+ loop {
+ let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
+
+ info!("{block_count} blocks found.");
+
+ let mut databases = Databases::import(config);
+ let mut datasets = Datasets::import(config)?;
+
+ iter_blocks(config, rpc, block_count, exit.clone(), &mut databases, &mut datasets)?;
+
+ if let Some(delay) = config.delay() {
+ sleep(Duration::from_secs(delay))
+ }
+
+ info!("Waiting for a new block...");
+
+ while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
+ sleep(Duration::from_secs(1))
+ }
+ }
+
+ // Ok(())
+}
diff --git a/parser/src/price/binance.rs b/_src/parser/price/binance.rs
similarity index 63%
rename from parser/src/price/binance.rs
rename to _src/parser/price/binance.rs
index bbd988b13..0967ad1e0 100644
--- a/parser/src/price/binance.rs
+++ b/_src/parser/price/binance.rs
@@ -1,26 +1,29 @@
#![allow(dead_code)]
-use std::{collections::BTreeMap, fs, path::Path};
+use std::{collections::BTreeMap, fs};
use color_eyre::eyre::ContextCompat;
use itertools::Itertools;
+use log::info;
use serde_json::Value;
use crate::{
- io::{Json, INPUTS_FOLDER_PATH},
- structs::{Date, Timestamp, OHLC},
- utils::{log, retry},
+ io::Json,
+ structs::{Config, Date, Timestamp, OHLC},
+ utils::retry,
};
pub struct Binance;
impl Binance {
- pub fn read_har_file() -> color_eyre::Result> {
- log("binance: read har file");
+ pub fn read_har_file(config: &Config) -> color_eyre::Result> {
+ info!("binance: read har file");
- fs::create_dir_all(INPUTS_FOLDER_PATH)?;
+ let path = config.path_inputs();
- let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har");
+ fs::create_dir_all(&path)?;
+
+ let path_binance_har = path.join("binance.har");
let json: BTreeMap = Json::import(&path_binance_har).unwrap_or_default();
@@ -104,7 +107,7 @@ impl Binance {
}
pub fn fetch_1mn_prices() -> color_eyre::Result> {
- log("binance: fetch 1mn");
+ info!("binance: fetch 1mn");
retry(
|_| {
@@ -117,33 +120,37 @@ impl Binance {
.as_array()
.context("Expect to be an array")?
.iter()
- .map(|value| {
+ .map(|value| -> color_eyre::Result<_> {
// [timestamp, open, high, low, close, volume, ...]
- let array = value.as_array().unwrap();
+ let array = value.as_array().context("Expect to be array")?;
- let timestamp = (array.first().unwrap().as_u64().unwrap() / 1_000) as u32;
+ let timestamp = (array
+ .first()
+ .context("Expect to have first")?
+ .as_u64()
+ .context("Expect to be convertible to u64")?
+ / 1_000) as u32;
- let get_f32 = |index: usize| {
- array
+ let get_f32 = |index: usize| -> color_eyre::Result {
+ Ok(array
.get(index)
- .unwrap()
+ .context("Expect to have index")?
.as_str()
- .unwrap()
- .parse::()
- .unwrap()
+ .context("Expect to have &str")?
+ .parse::()?)
};
- (
+ Ok((
timestamp,
OHLC {
- open: get_f32(1),
- high: get_f32(2),
- low: get_f32(3),
- close: get_f32(4),
+ open: get_f32(1)?,
+ high: get_f32(2)?,
+ low: get_f32(3)?,
+ close: get_f32(4)?,
},
- )
+ ))
})
- .collect::>())
+ .collect::, _>>()?)
},
30,
10,
@@ -151,7 +158,7 @@ impl Binance {
}
pub fn fetch_daily_prices() -> color_eyre::Result> {
- log("binance: fetch 1d");
+ info!("binance: fetch 1d");
retry(
|_| {
@@ -164,36 +171,40 @@ impl Binance {
.as_array()
.context("Expect to be an array")?
.iter()
- .map(|value| {
+ .map(|value| -> color_eyre::Result<_> {
// [timestamp, open, high, low, close, volume, ...]
- let array = value.as_array().unwrap();
+ let array = value.as_array().context("Expect to be array")?;
- let date = Timestamp::wrap(
- (array.first().unwrap().as_u64().unwrap() / 1_000) as u32,
+ let date = Timestamp::from(
+ (array
+ .first()
+ .context("Expect to have first")?
+ .as_u64()
+ .context("Expect to be convertible to u64")?
+ / 1_000) as u32,
)
.to_date();
- let get_f32 = |index: usize| {
- array
+ let get_f32 = |index: usize| -> color_eyre::Result {
+ Ok(array
.get(index)
- .unwrap()
+ .context("Expect to have index")?
.as_str()
- .unwrap()
- .parse::()
- .unwrap()
+ .context("Expect to have &str")?
+ .parse::()?)
};
- (
+ Ok((
date,
OHLC {
- open: get_f32(1),
- high: get_f32(2),
- low: get_f32(3),
- close: get_f32(4),
+ open: get_f32(1)?,
+ high: get_f32(2)?,
+ low: get_f32(3)?,
+ close: get_f32(4)?,
},
- )
+ ))
})
- .collect::>())
+ .collect::, _>>()?)
},
30,
10,
diff --git a/parser/src/price/kibo.rs b/_src/parser/price/kibo.rs
similarity index 70%
rename from parser/src/price/kibo.rs
rename to _src/parser/price/kibo.rs
index 51a48adc0..858b7d7ac 100644
--- a/parser/src/price/kibo.rs
+++ b/_src/parser/price/kibo.rs
@@ -2,13 +2,12 @@ use std::{collections::BTreeMap, str::FromStr};
use chrono::NaiveDate;
use color_eyre::eyre::ContextCompat;
-use itertools::Itertools;
+use log::info;
use serde_json::Value;
use crate::{
- structs::{Date, DateMapChunkId, HeightMapChunkId, OHLC},
- utils::{log, retry},
- MapChunkId,
+ structs::{Date, DateMapChunkId, HeightMapChunkId, MapChunkId, OHLC},
+ utils::retry,
};
pub struct Kibo;
@@ -28,7 +27,7 @@ impl Kibo {
}
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result> {
- log("kibo: fetch height prices");
+ info!("kibo: fetch height prices");
retry(
|try_index| {
@@ -40,7 +39,7 @@ impl Kibo {
))?
.json()?;
- Ok(body
+ let vec = body
.as_object()
.context("Expect to be an object")?
.get("dataset")
@@ -53,7 +52,9 @@ impl Kibo {
.context("Expect to be an array")?
.iter()
.map(Self::value_to_ohlc)
- .collect_vec())
+ .collect::, _>>()?;
+
+ Ok(vec)
},
30,
RETRIES,
@@ -61,7 +62,7 @@ impl Kibo {
}
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result> {
- log("kibo: fetch date prices");
+ info!("kibo: fetch date prices");
retry(
|try_index| {
@@ -85,28 +86,33 @@ impl Kibo {
.as_object()
.context("Expect to be an object")?
.iter()
- .map(|(serialized_date, value)| {
- let date = Date::wrap(NaiveDate::from_str(serialized_date).unwrap());
-
- (date, Self::value_to_ohlc(value))
+ .map(|(serialized_date, value)| -> color_eyre::Result<_> {
+ let date = Date::wrap(NaiveDate::from_str(serialized_date)?);
+ Ok((date, Self::value_to_ohlc(value)?))
})
- .collect::>())
+ .collect::, _>>()?)
},
30,
RETRIES,
)
}
- fn value_to_ohlc(value: &Value) -> OHLC {
- let ohlc = value.as_object().unwrap();
+ fn value_to_ohlc(value: &Value) -> color_eyre::Result {
+ let ohlc = value.as_object().context("Expect as_object to work")?;
- let get_value = |key: &str| ohlc.get(key).unwrap().as_f64().unwrap() as f32;
+ let get_value = |key: &str| -> color_eyre::Result {
+ Ok(ohlc
+ .get(key)
+ .context("Expect get key to work")?
+ .as_f64()
+ .context("Expect as_f64 to work")? as f32)
+ };
- OHLC {
- open: get_value("open"),
- high: get_value("high"),
- low: get_value("low"),
- close: get_value("close"),
- }
+ Ok(OHLC {
+ open: get_value("open")?,
+ high: get_value("high")?,
+ low: get_value("low")?,
+ close: get_value("close")?,
+ })
}
}
diff --git a/parser/src/price/kraken.rs b/_src/parser/price/kraken.rs
similarity index 51%
rename from parser/src/price/kraken.rs
rename to _src/parser/price/kraken.rs
index 7567964b1..99566fee0 100644
--- a/parser/src/price/kraken.rs
+++ b/_src/parser/price/kraken.rs
@@ -1,18 +1,19 @@
use std::collections::BTreeMap;
use color_eyre::eyre::ContextCompat;
+use log::info;
use serde_json::Value;
use crate::{
structs::{Date, Timestamp, OHLC},
- utils::{log, retry},
+ utils::retry,
};
pub struct Kraken;
impl Kraken {
pub fn fetch_1mn_prices() -> color_eyre::Result> {
- log("kraken: fetch 1mn");
+ info!("kraken: fetch 1mn");
retry(
|_| {
@@ -33,32 +34,36 @@ impl Kraken {
.as_array()
.context("Expect to be an array")?
.iter()
- .map(|value| {
- let array = value.as_array().unwrap();
+ .map(|value| -> color_eyre::Result<_> {
+ let array = value.as_array().context("Expect as_array to work")?;
- let timestamp = array.first().unwrap().as_u64().unwrap() as u32;
+ let timestamp = array
+ .first()
+ .context("Expect first to work")?
+ .as_u64()
+ .expect("Expect as_u64 to work")
+ as u32;
- let get_f32 = |index: usize| {
- array
+ let get_f32 = |index: usize| -> color_eyre::Result {
+ Ok(array
.get(index)
- .unwrap()
+ .context("Expect get index to work")?
.as_str()
- .unwrap()
- .parse::()
- .unwrap()
+ .context("Expect as_str to work")?
+ .parse::()?)
};
- (
+ Ok((
timestamp,
OHLC {
- open: get_f32(1),
- high: get_f32(2),
- low: get_f32(3),
- close: get_f32(4),
+ open: get_f32(1)?,
+ high: get_f32(2)?,
+ low: get_f32(3)?,
+ close: get_f32(4)?,
},
- )
+ ))
})
- .collect::>())
+ .collect::, _>>()?)
},
30,
10,
@@ -66,7 +71,7 @@ impl Kraken {
}
pub fn fetch_daily_prices() -> color_eyre::Result> {
- log("fetch kraken daily");
+ info!("fetch kraken daily");
retry(
|_| {
@@ -87,33 +92,39 @@ impl Kraken {
.as_array()
.context("Expect to be an array")?
.iter()
- .map(|value| {
- let array = value.as_array().unwrap();
+ .map(|value| -> color_eyre::Result<_> {
+ let array = value.as_array().context("Expect as_array to work")?;
- let date = Timestamp::wrap(array.first().unwrap().as_u64().unwrap() as u32)
- .to_date();
-
- let get_f32 = |index: usize| {
+ let date = Timestamp::from(
array
+ .first()
+ .context("Expect first to work")?
+ .as_u64()
+ .context("Expect as_u64 to work")?
+ as u32,
+ )
+ .to_date();
+
+ let get_f32 = |index: usize| -> color_eyre::Result {
+ Ok(array
.get(index)
- .unwrap()
+ .context("Expect get index to work")?
.as_str()
- .unwrap()
- .parse::()
- .unwrap()
+ .context("Expect as_str to work")?
+ .parse::()?)
};
- (
+ Ok((
date,
OHLC {
- open: get_f32(1),
- high: get_f32(2),
- low: get_f32(3),
- close: get_f32(4),
+ open: get_f32(1)?,
+ high: get_f32(2)?,
+ low: get_f32(3)?,
+ close: get_f32(4)?,
},
- )
+ ))
})
- .collect::>())
+ .collect::, _>>()?)
},
30,
10,
diff --git a/parser/src/price/mod.rs b/_src/parser/price/mod.rs
similarity index 100%
rename from parser/src/price/mod.rs
rename to _src/parser/price/mod.rs
diff --git a/_src/parser/states/_trait.rs b/_src/parser/states/_trait.rs
new file mode 100644
index 000000000..c07277890
--- /dev/null
+++ b/_src/parser/states/_trait.rs
@@ -0,0 +1,36 @@
+use std::{
+ fmt::Debug,
+ fs, io,
+ path::{Path, PathBuf},
+};
+
+use bincode::{Decode, Encode};
+use serde::{de::DeserializeOwned, Serialize};
+
+use crate::{io::Serialization, structs::Config};
+
+pub trait AnyState
+where
+ Self: Debug + Encode + Decode + Serialize + DeserializeOwned,
+{
+ fn name<'a>() -> &'a str;
+
+ fn path(config: &Config) -> PathBuf {
+ config.path_states().join(Self::name())
+ }
+
+ fn reset(&mut self, config: &Config) -> color_eyre::Result<(), io::Error> {
+ self.clear();
+ fs::remove_file(Self::path(config))
+ }
+
+ fn import(config: &Config) -> color_eyre::Result {
+ Serialization::Binary.import(&Self::path(config))
+ }
+
+ fn export(&self, config: &Config) -> color_eyre::Result<()> {
+ Serialization::Binary.export(Path::new(&Self::path(config)), self)
+ }
+
+ fn clear(&mut self);
+}
diff --git a/parser/src/states/cohorts_states/address/cohort_durable_states.rs b/_src/parser/states/cohorts_states/address/cohort_durable_states.rs
similarity index 97%
rename from parser/src/states/cohorts_states/address/cohort_durable_states.rs
rename to _src/parser/states/cohorts_states/address/cohort_durable_states.rs
index 55c367ce8..8bcbb201b 100644
--- a/parser/src/states/cohorts_states/address/cohort_durable_states.rs
+++ b/_src/parser/states/cohorts_states/address/cohort_durable_states.rs
@@ -3,7 +3,7 @@ use std::ops::AddAssign;
use allocative::Allocative;
use crate::{
- states::{DurableStates, IsZero, OneShotStates, PriceToValue, UnrealizedState},
+ parser::states::{DurableStates, IsZero, OneShotStates, PriceToValue, UnrealizedState},
structs::{Amount, Price},
};
diff --git a/parser/src/states/cohorts_states/address/cohort_id.rs b/_src/parser/states/cohorts_states/address/cohort_id.rs
similarity index 100%
rename from parser/src/states/cohorts_states/address/cohort_id.rs
rename to _src/parser/states/cohorts_states/address/cohort_id.rs
diff --git a/parser/src/states/cohorts_states/address/cohorts_durable_states.rs b/_src/parser/states/cohorts_states/address/cohorts_durable_states.rs
similarity index 99%
rename from parser/src/states/cohorts_states/address/cohorts_durable_states.rs
rename to _src/parser/states/cohorts_states/address/cohorts_durable_states.rs
index 932143461..0dece4f9d 100644
--- a/parser/src/states/cohorts_states/address/cohorts_durable_states.rs
+++ b/_src/parser/states/cohorts_states/address/cohorts_durable_states.rs
@@ -5,7 +5,7 @@ use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::{
- databases::AddressIndexToAddressData,
+ parser::databases::AddressIndexToAddressData,
structs::{AddressData, AddressRealizedData, Amount, Price},
};
diff --git a/parser/src/states/cohorts_states/address/cohorts_input_states.rs b/_src/parser/states/cohorts_states/address/cohorts_input_states.rs
similarity index 97%
rename from parser/src/states/cohorts_states/address/cohorts_input_states.rs
rename to _src/parser/states/cohorts_states/address/cohorts_input_states.rs
index 5f6ef9afa..0da4376c5 100644
--- a/parser/src/states/cohorts_states/address/cohorts_input_states.rs
+++ b/_src/parser/states/cohorts_states/address/cohorts_input_states.rs
@@ -1,7 +1,7 @@
use derive_deref::{Deref, DerefMut};
use crate::{
- states::InputState,
+ parser::states::InputState,
structs::{AddressRealizedData, Amount, LiquidityClassification},
};
diff --git a/parser/src/states/cohorts_states/address/cohorts_one_shot_states.rs b/_src/parser/states/cohorts_states/address/cohorts_one_shot_states.rs
similarity index 81%
rename from parser/src/states/cohorts_states/address/cohorts_one_shot_states.rs
rename to _src/parser/states/cohorts_states/address/cohorts_one_shot_states.rs
index 042e44ea4..bda560fa6 100644
--- a/parser/src/states/cohorts_states/address/cohorts_one_shot_states.rs
+++ b/_src/parser/states/cohorts_states/address/cohorts_one_shot_states.rs
@@ -1,6 +1,6 @@
use derive_deref::{Deref, DerefMut};
-use crate::states::OneShotStates;
+use crate::parser::states::OneShotStates;
use super::SplitByAddressCohort;
diff --git a/parser/src/states/cohorts_states/address/cohorts_output_states.rs b/_src/parser/states/cohorts_states/address/cohorts_output_states.rs
similarity index 97%
rename from parser/src/states/cohorts_states/address/cohorts_output_states.rs
rename to _src/parser/states/cohorts_states/address/cohorts_output_states.rs
index d0a682129..155ef98d2 100644
--- a/parser/src/states/cohorts_states/address/cohorts_output_states.rs
+++ b/_src/parser/states/cohorts_states/address/cohorts_output_states.rs
@@ -1,7 +1,7 @@
use derive_deref::{Deref, DerefMut};
use crate::{
- states::OutputState,
+ parser::states::OutputState,
structs::{AddressRealizedData, Amount, LiquidityClassification},
};
diff --git a/parser/src/states/cohorts_states/address/cohorts_realized_states.rs b/_src/parser/states/cohorts_states/address/cohorts_realized_states.rs
similarity index 98%
rename from parser/src/states/cohorts_states/address/cohorts_realized_states.rs
rename to _src/parser/states/cohorts_states/address/cohorts_realized_states.rs
index 524cc31f6..22d2a228b 100644
--- a/parser/src/states/cohorts_states/address/cohorts_realized_states.rs
+++ b/_src/parser/states/cohorts_states/address/cohorts_realized_states.rs
@@ -1,7 +1,7 @@
use derive_deref::{Deref, DerefMut};
use crate::{
- states::RealizedState,
+ parser::states::RealizedState,
structs::{AddressRealizedData, LiquidityClassification, Price},
};
diff --git a/parser/src/states/cohorts_states/address/mod.rs b/_src/parser/states/cohorts_states/address/mod.rs
similarity index 100%
rename from parser/src/states/cohorts_states/address/mod.rs
rename to _src/parser/states/cohorts_states/address/mod.rs
diff --git a/parser/src/states/cohorts_states/address/split_by_address_cohort.rs b/_src/parser/states/cohorts_states/address/split_by_address_cohort.rs
similarity index 100%
rename from parser/src/states/cohorts_states/address/split_by_address_cohort.rs
rename to _src/parser/states/cohorts_states/address/split_by_address_cohort.rs
diff --git a/parser/src/states/cohorts_states/any/capitalization_state.rs b/_src/parser/states/cohorts_states/any/capitalization_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/capitalization_state.rs
rename to _src/parser/states/cohorts_states/any/capitalization_state.rs
diff --git a/parser/src/states/cohorts_states/any/durable_states.rs b/_src/parser/states/cohorts_states/any/durable_states.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/durable_states.rs
rename to _src/parser/states/cohorts_states/any/durable_states.rs
diff --git a/parser/src/states/cohorts_states/any/input_state.rs b/_src/parser/states/cohorts_states/any/input_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/input_state.rs
rename to _src/parser/states/cohorts_states/any/input_state.rs
diff --git a/parser/src/states/cohorts_states/any/mod.rs b/_src/parser/states/cohorts_states/any/mod.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/mod.rs
rename to _src/parser/states/cohorts_states/any/mod.rs
diff --git a/parser/src/states/cohorts_states/any/one_shot_states.rs b/_src/parser/states/cohorts_states/any/one_shot_states.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/one_shot_states.rs
rename to _src/parser/states/cohorts_states/any/one_shot_states.rs
diff --git a/parser/src/states/cohorts_states/any/output_state.rs b/_src/parser/states/cohorts_states/any/output_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/output_state.rs
rename to _src/parser/states/cohorts_states/any/output_state.rs
diff --git a/parser/src/states/cohorts_states/any/price_paid_state.rs b/_src/parser/states/cohorts_states/any/price_paid_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/price_paid_state.rs
rename to _src/parser/states/cohorts_states/any/price_paid_state.rs
diff --git a/parser/src/states/cohorts_states/any/price_to_value.rs b/_src/parser/states/cohorts_states/any/price_to_value.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/price_to_value.rs
rename to _src/parser/states/cohorts_states/any/price_to_value.rs
diff --git a/parser/src/states/cohorts_states/any/realized_state.rs b/_src/parser/states/cohorts_states/any/realized_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/realized_state.rs
rename to _src/parser/states/cohorts_states/any/realized_state.rs
diff --git a/parser/src/states/cohorts_states/any/supply_state.rs b/_src/parser/states/cohorts_states/any/supply_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/supply_state.rs
rename to _src/parser/states/cohorts_states/any/supply_state.rs
diff --git a/parser/src/states/cohorts_states/any/unrealized_state.rs b/_src/parser/states/cohorts_states/any/unrealized_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/unrealized_state.rs
rename to _src/parser/states/cohorts_states/any/unrealized_state.rs
diff --git a/parser/src/states/cohorts_states/any/utxo_state.rs b/_src/parser/states/cohorts_states/any/utxo_state.rs
similarity index 100%
rename from parser/src/states/cohorts_states/any/utxo_state.rs
rename to _src/parser/states/cohorts_states/any/utxo_state.rs
diff --git a/parser/src/states/cohorts_states/mod.rs b/_src/parser/states/cohorts_states/mod.rs
similarity index 100%
rename from parser/src/states/cohorts_states/mod.rs
rename to _src/parser/states/cohorts_states/mod.rs
diff --git a/parser/src/states/cohorts_states/utxo/cohort_durable_states.rs b/_src/parser/states/cohorts_states/utxo/cohort_durable_states.rs
similarity index 97%
rename from parser/src/states/cohorts_states/utxo/cohort_durable_states.rs
rename to _src/parser/states/cohorts_states/utxo/cohort_durable_states.rs
index af47446e4..13ccce07a 100644
--- a/parser/src/states/cohorts_states/utxo/cohort_durable_states.rs
+++ b/_src/parser/states/cohorts_states/utxo/cohort_durable_states.rs
@@ -1,7 +1,7 @@
use allocative::Allocative;
use crate::{
- states::{DurableStates, OneShotStates, PriceToValue, UnrealizedState},
+ parser::states::{DurableStates, OneShotStates, PriceToValue, UnrealizedState},
structs::{Amount, Price},
};
diff --git a/parser/src/states/cohorts_states/utxo/cohort_filter.rs b/_src/parser/states/cohorts_states/utxo/cohort_filter.rs
similarity index 100%
rename from parser/src/states/cohorts_states/utxo/cohort_filter.rs
rename to _src/parser/states/cohorts_states/utxo/cohort_filter.rs
diff --git a/parser/src/states/cohorts_states/utxo/cohort_filters.rs b/_src/parser/states/cohorts_states/utxo/cohort_filters.rs
similarity index 100%
rename from parser/src/states/cohorts_states/utxo/cohort_filters.rs
rename to _src/parser/states/cohorts_states/utxo/cohort_filters.rs
diff --git a/parser/src/states/cohorts_states/utxo/cohort_id.rs b/_src/parser/states/cohorts_states/utxo/cohort_id.rs
similarity index 100%
rename from parser/src/states/cohorts_states/utxo/cohort_id.rs
rename to _src/parser/states/cohorts_states/utxo/cohort_id.rs
diff --git a/parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs b/_src/parser/states/cohorts_states/utxo/cohorts_durable_states.rs
similarity index 99%
rename from parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs
rename to _src/parser/states/cohorts_states/utxo/cohorts_durable_states.rs
index 16cafddcf..2ba92dbf2 100644
--- a/parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs
+++ b/_src/parser/states/cohorts_states/utxo/cohorts_durable_states.rs
@@ -3,7 +3,7 @@ use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::{
- states::DateDataVec,
+ parser::states::DateDataVec,
structs::{Amount, BlockData, Price, SentData, Timestamp},
};
diff --git a/parser/src/states/cohorts_states/utxo/cohorts_one_shot_states.rs b/_src/parser/states/cohorts_states/utxo/cohorts_one_shot_states.rs
similarity index 81%
rename from parser/src/states/cohorts_states/utxo/cohorts_one_shot_states.rs
rename to _src/parser/states/cohorts_states/utxo/cohorts_one_shot_states.rs
index efd4db6a5..328fddf7a 100644
--- a/parser/src/states/cohorts_states/utxo/cohorts_one_shot_states.rs
+++ b/_src/parser/states/cohorts_states/utxo/cohorts_one_shot_states.rs
@@ -1,6 +1,6 @@
use derive_deref::{Deref, DerefMut};
-use crate::states::OneShotStates;
+use crate::parser::states::OneShotStates;
use super::SplitByUTXOCohort;
diff --git a/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs b/_src/parser/states/cohorts_states/utxo/cohorts_sent_states.rs
similarity index 98%
rename from parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs
rename to _src/parser/states/cohorts_states/utxo/cohorts_sent_states.rs
index 9b0da8a05..f09d908f7 100644
--- a/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs
+++ b/_src/parser/states/cohorts_states/utxo/cohorts_sent_states.rs
@@ -3,7 +3,7 @@ use std::{cmp::Ordering, collections::BTreeMap};
use derive_deref::{Deref, DerefMut};
use crate::{
- states::{DateDataVec, InputState, RealizedState},
+ parser::states::{DateDataVec, InputState, RealizedState},
structs::{BlockPath, Price, SentData, Timestamp},
};
diff --git a/parser/src/states/cohorts_states/utxo/mod.rs b/_src/parser/states/cohorts_states/utxo/mod.rs
similarity index 100%
rename from parser/src/states/cohorts_states/utxo/mod.rs
rename to _src/parser/states/cohorts_states/utxo/mod.rs
diff --git a/parser/src/states/cohorts_states/utxo/split_by_utxo_cohort.rs b/_src/parser/states/cohorts_states/utxo/split_by_utxo_cohort.rs
similarity index 100%
rename from parser/src/states/cohorts_states/utxo/split_by_utxo_cohort.rs
rename to _src/parser/states/cohorts_states/utxo/split_by_utxo_cohort.rs
diff --git a/parser/src/states/counters.rs b/_src/parser/states/counters.rs
similarity index 100%
rename from parser/src/states/counters.rs
rename to _src/parser/states/counters.rs
diff --git a/parser/src/states/date_data_vec.rs b/_src/parser/states/date_data_vec.rs
similarity index 100%
rename from parser/src/states/date_data_vec.rs
rename to _src/parser/states/date_data_vec.rs
diff --git a/parser/src/states/mod.rs b/_src/parser/states/mod.rs
similarity index 54%
rename from parser/src/states/mod.rs
rename to _src/parser/states/mod.rs
index 878b6d672..1558b0005 100644
--- a/parser/src/states/mod.rs
+++ b/_src/parser/states/mod.rs
@@ -1,4 +1,4 @@
-use std::thread;
+use std::{fs, thread};
mod _trait;
mod cohorts_states;
@@ -11,8 +11,9 @@ use allocative::Allocative;
pub use cohorts_states::*;
use counters::*;
use date_data_vec::*;
+use log::info;
-use crate::utils::log;
+use crate::structs::Config;
#[derive(Default, Allocative)]
pub struct States {
@@ -23,12 +24,12 @@ pub struct States {
}
impl States {
- pub fn import() -> color_eyre::Result {
- let date_data_vec_handle = thread::spawn(DateDataVec::import);
+ pub fn import(config: &Config) -> color_eyre::Result {
+ fs::create_dir_all(config.path_states())?;
- let address_counters = Counters::import()?;
+ let date_data_vec = DateDataVec::import(config)?;
- let date_data_vec = date_data_vec_handle.join().unwrap()?;
+ let address_counters = Counters::import(config)?;
Ok(Self {
address_cohorts_durable_states: None,
@@ -38,24 +39,24 @@ impl States {
})
}
- pub fn reset(&mut self, include_addresses: bool) {
- log("Reseting all states...");
+ pub fn reset(&mut self, config: &Config, include_addresses: bool) {
+ info!("Reseting all states...");
- let _ = self.date_data_vec.reset();
+ let _ = self.date_data_vec.reset(config);
self.utxo_cohorts_durable_states = None;
if include_addresses {
- let _ = self.address_counters.reset();
+ let _ = self.address_counters.reset(config);
self.address_cohorts_durable_states = None;
}
}
- pub fn export(&self) -> color_eyre::Result<()> {
+ pub fn export(&self, config: &Config) -> color_eyre::Result<()> {
thread::scope(|s| {
- s.spawn(|| self.address_counters.export().unwrap());
- s.spawn(|| self.date_data_vec.export().unwrap());
+ s.spawn(|| self.address_counters.export(config).unwrap());
+ s.spawn(|| self.date_data_vec.export(config).unwrap());
});
Ok(())
diff --git a/parser/src/structs/address.rs b/_src/structs/address.rs
similarity index 96%
rename from parser/src/structs/address.rs
rename to _src/structs/address.rs
index 7fc0255d5..da6dc5d4a 100644
--- a/parser/src/structs/address.rs
+++ b/_src/structs/address.rs
@@ -1,5 +1,5 @@
-use bitcoin_hashes::{hash160, Hash};
-use biter::bitcoin::TxOut;
+use bitcoin_hashes::hash160;
+use brk_parser::bitcoin::TxOut;
use super::{AddressType, Counter, U8x19, U8x31};
@@ -56,7 +56,7 @@ impl Address {
let hash = hash160::Hash::hash(pk);
- let (prefix, rest) = Self::split_slice(&hash[..]);
+ let (prefix, rest) = Self::split_slice(&hash.as_byte_array()[..]);
Self::P2PK((prefix, rest.into()))
} else if script.is_p2pkh() {
diff --git a/parser/src/structs/address_data.rs b/_src/structs/address_data.rs
similarity index 97%
rename from parser/src/structs/address_data.rs
rename to _src/structs/address_data.rs
index 3d0968d0a..636819317 100644
--- a/parser/src/structs/address_data.rs
+++ b/_src/structs/address_data.rs
@@ -1,6 +1,6 @@
use allocative::Allocative;
use color_eyre::eyre::eyre;
-use sanakirja::{direct_repr, Storable, UnsizedStorable};
+use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::{AddressType, Amount, EmptyAddressData, LiquidityClassification, Price};
diff --git a/parser/src/structs/address_liquidity.rs b/_src/structs/address_liquidity.rs
similarity index 100%
rename from parser/src/structs/address_liquidity.rs
rename to _src/structs/address_liquidity.rs
diff --git a/parser/src/structs/address_realized_data.rs b/_src/structs/address_realized_data.rs
similarity index 100%
rename from parser/src/structs/address_realized_data.rs
rename to _src/structs/address_realized_data.rs
diff --git a/parser/src/structs/address_size.rs b/_src/structs/address_size.rs
similarity index 100%
rename from parser/src/structs/address_size.rs
rename to _src/structs/address_size.rs
diff --git a/parser/src/structs/address_split.rs b/_src/structs/address_split.rs
similarity index 100%
rename from parser/src/structs/address_split.rs
rename to _src/structs/address_split.rs
diff --git a/parser/src/structs/address_type.rs b/_src/structs/address_type.rs
similarity index 100%
rename from parser/src/structs/address_type.rs
rename to _src/structs/address_type.rs
diff --git a/parser/src/structs/amount.rs b/_src/structs/amount.rs
similarity index 91%
rename from parser/src/structs/amount.rs
rename to _src/structs/amount.rs
index 6ff8ce1d4..5a29adf42 100644
--- a/parser/src/structs/amount.rs
+++ b/_src/structs/amount.rs
@@ -5,32 +5,19 @@ use std::{
use allocative::{Allocative, Visitor};
use bincode::{
+ BorrowDecode, Decode, Encode,
de::{BorrowDecoder, Decoder},
enc::Encoder,
error::{DecodeError, EncodeError},
- BorrowDecode, Decode, Encode,
};
-use biter::bitcoin::Amount as BitcoinAmount;
+use brk_parser::bitcoin::Amount as BitcoinAmount;
use derive_deref::{Deref, DerefMut};
-use sanakirja::{direct_repr, Storable, UnsizedStorable};
use serde::{Deserialize, Serialize};
+use snkrj::{Storable, UnsizedStorable, direct_repr};
use super::Height;
-#[derive(
- Debug,
- PartialEq,
- Eq,
- PartialOrd,
- Ord,
- Clone,
- Copy,
- Deref,
- DerefMut,
- Default,
- Serialize,
- Deserialize,
-)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default, Serialize, Deserialize)]
pub struct Amount(BitcoinAmount);
direct_repr!(Amount);
diff --git a/_src/structs/any_map.rs b/_src/structs/any_map.rs
new file mode 100644
index 000000000..371898295
--- /dev/null
+++ b/_src/structs/any_map.rs
@@ -0,0 +1,23 @@
+use std::path::Path;
+
+use serde_json::Value;
+
+use crate::io::Serialization;
+
+use super::{Config, MapKind, MapPath};
+
+pub trait AnyMap {
+ fn path(&self) -> &Path;
+ fn path_parent(&self) -> &Path;
+ fn path_last(&self) -> &Option;
+ fn last_value(&self) -> Option;
+ fn serialization(&self) -> Serialization;
+ fn type_name(&self) -> &str;
+ fn key_name(&self) -> &str;
+ fn pre_export(&mut self);
+ fn export(&self) -> color_eyre::Result<()>;
+ fn post_export(&mut self);
+ fn delete_files(&self);
+ fn kind(&self) -> MapKind;
+ fn id(&self, config: &Config) -> String;
+}
diff --git a/parser/src/structs/array.rs b/_src/structs/array.rs
similarity index 92%
rename from parser/src/structs/array.rs
rename to _src/structs/array.rs
index dbfb67592..8a915db6d 100644
--- a/parser/src/structs/array.rs
+++ b/_src/structs/array.rs
@@ -2,7 +2,7 @@ use std::fmt::Debug;
use allocative::Allocative;
use derive_deref::{Deref, DerefMut};
-use sanakirja::{direct_repr, Storable, UnsizedStorable};
+use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative,
diff --git a/parser/src/structs/bi_map.rs b/_src/structs/bi_map.rs
similarity index 97%
rename from parser/src/structs/bi_map.rs
rename to _src/structs/bi_map.rs
index dd271ec14..733c06846 100644
--- a/parser/src/structs/bi_map.rs
+++ b/_src/structs/bi_map.rs
@@ -8,10 +8,10 @@ use allocative::Allocative;
use crate::utils::{LossyFrom, TARGET_BLOCKS_PER_DAY};
use super::{
- AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapKind, MapValue,
+ AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapKind, MapPath, MapValue,
};
-#[derive(Allocative)]
+#[derive(Allocative, Debug)]
pub struct BiMap
where
Value: MapValue,
@@ -24,14 +24,14 @@ impl BiMap
where
Value: MapValue,
{
- pub fn new_bin(version: u32, kind: MapKind, path: &str) -> Self {
+ pub fn new_bin(version: u32, kind: MapKind, path: &MapPath) -> Self {
Self {
height: HeightMap::_new_bin(version, kind, path, true),
date: DateMap::_new_bin(version, kind, path, false),
}
}
- pub fn new_json(version: u32, kind: MapKind, path: &str) -> Self {
+ pub fn new_json(version: u32, kind: MapKind, path: &MapPath) -> Self {
Self {
height: HeightMap::new_json(version, kind, path, true),
date: DateMap::new_json(version, kind, path, false),
diff --git a/parser/src/structs/block_data.rs b/_src/structs/block_data.rs
similarity index 100%
rename from parser/src/structs/block_data.rs
rename to _src/structs/block_data.rs
diff --git a/parser/src/structs/block_path.rs b/_src/structs/block_path.rs
similarity index 100%
rename from parser/src/structs/block_path.rs
rename to _src/structs/block_path.rs
diff --git a/_src/structs/config.rs b/_src/structs/config.rs
new file mode 100644
index 000000000..683ec1181
--- /dev/null
+++ b/_src/structs/config.rs
@@ -0,0 +1,326 @@
+use std::{
+ fs::{self},
+ mem,
+ path::{Path, PathBuf},
+};
+
+use brk_parser::bitcoincore_rpc::Auth;
+use clap::Parser;
+use color_eyre::eyre::eyre;
+use log::info;
+use serde::{Deserialize, Serialize};
+
+use crate::io::JSON_EXTENSION;
+
+use super::MapPath;
+
+#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
+#[command(version, about, long_about = None)]
+pub struct Config {
+ /// Bitcoin data directory path, saved
+ #[arg(long, value_name = "PATH")]
+ bitcoindir: Option,
+
+ /// Kibo output directory path, saved
+ #[arg(long, value_name = "PATH")]
+ kibodir: Option,
+
+ /// Bitcoin RPC ip, default: localhost, saved
+ #[arg(long, value_name = "IP")]
+ rpcconnect: Option,
+
+ /// Bitcoin RPC port, default: 8332, saved
+ #[arg(long, value_name = "PORT")]
+ rpcport: Option,
+
+ /// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
+ #[arg(long, value_name = "PATH")]
+ rpccookiefile: Option,
+
+ /// Bitcoin RPC username, saved
+ #[arg(long, value_name = "USERNAME")]
+ rpcuser: Option,
+
+ /// Bitcoin RPC password, saved
+ #[arg(long, value_name = "PASSWORD")]
+ rpcpassword: Option,
+
+ /// Delay between runs, default: 0, saved
+ #[arg(long, value_name = "SECONDS")]
+ delay: Option,
+
+ /// Disable the parser, not saved
+ #[serde(default)]
+ #[arg(long, default_value_t = false)]
+ no_parser: bool,
+
+ /// Disable the server, not saved
+ #[serde(default)]
+ #[arg(long, default_value_t = false)]
+ no_server: bool,
+
+ /// Run without saving, not saved
+ #[serde(default)]
+ #[arg(long, default_value_t = false)]
+ dry_run: bool,
+
+ /// Record ram usage, not saved
+ #[serde(default)]
+ #[arg(long, default_value_t = false)]
+ record_ram_usage: bool,
+
+ /// Recompute all computed datasets, not saved
+ #[serde(default)]
+ #[arg(long, default_value_t = false)]
+ recompute_computed: bool,
+}
+
+impl Config {
+ pub const DATASET_DIR_NAME: &str = "datasets";
+ pub const DATABASES_DIR_NAME: &str = "databases";
+
+ pub fn import() -> color_eyre::Result {
+ let path = Self::path_dot_kibo();
+ let _ = fs::create_dir_all(&path);
+
+ let path = path.join("config.toml");
+
+ let mut config_saved = Self::read(&path);
+
+ let mut config_args = Config::parse();
+
+ if let Some(bitcoindir) = config_args.bitcoindir.take() {
+ config_saved.bitcoindir = Some(bitcoindir);
+ }
+
+ if let Some(kibodir) = config_args.kibodir.take() {
+ config_saved.kibodir = Some(kibodir);
+ }
+
+ if let Some(rpcconnect) = config_args.rpcconnect.take() {
+ config_saved.rpcconnect = Some(rpcconnect);
+ }
+
+ if let Some(rpcport) = config_args.rpcport.take() {
+ config_saved.rpcport = Some(rpcport);
+ }
+
+ if let Some(rpccookiefile) = config_args.rpccookiefile.take() {
+ config_saved.rpccookiefile = Some(rpccookiefile);
+ }
+
+ if let Some(rpcuser) = config_args.rpcuser.take() {
+ config_saved.rpcuser = Some(rpcuser);
+ }
+
+ if let Some(rpcpassword) = config_args.rpcpassword.take() {
+ config_saved.rpcpassword = Some(rpcpassword);
+ }
+
+ if let Some(delay) = config_args.delay.take() {
+ config_saved.delay = Some(delay);
+ }
+
+ // if let Some(max_ram) = config_args.max_ram.take() {
+ // config_saved.max_ram = Some(max_ram);
+ // }
+
+ // Done importing
+
+ let mut config = config_saved;
+
+ config.check();
+
+ config.write(&path)?;
+
+ config.no_parser = mem::take(&mut config_args.no_parser);
+ config.no_server = mem::take(&mut config_args.no_server);
+ config.dry_run = mem::take(&mut config_args.dry_run);
+ config.record_ram_usage = mem::take(&mut config_args.record_ram_usage);
+ config.recompute_computed = mem::take(&mut config_args.recompute_computed);
+
+ info!("Configuration {{");
+ info!(" bitcoindir: {:?}", config.bitcoindir);
+ info!(" kibodir: {:?}", config.kibodir);
+ info!(" rpcconnect: {:?}", config.rpcconnect);
+ info!(" rpcport: {:?}", config.rpcport);
+ info!(" rpccookiefile: {:?}", config.rpccookiefile);
+ info!(" rpcuser: {:?}", config.rpcuser);
+ info!(" rpcpassword: {:?}", config.rpcpassword);
+ info!(" delay: {:?}", config.delay);
+ // info!(" max_ram: {:?}", config.max_ram);
+ info!(" parser: {:?}", config.parser());
+ info!(" server: {:?}", config.server());
+ info!(" dry_run: {:?}", config.dry_run());
+ info!(" record_ram_usage: {:?}", config.record_ram_usage());
+ info!(" recompute_computed: {:?}", config.recompute_computed());
+ info!("}}");
+
+ if config_args != Config::default() {
+ dbg!(config_args);
+ panic!("Didn't consume the full config")
+ }
+
+ Ok(config)
+ }
+
+ fn check(&self) {
+ if self.bitcoindir.is_none() {
+ println!(
+ "You need to set the --bitcoindir parameter at least once to run the parser.\nRun the program with '-h' for help."
+ );
+ std::process::exit(1);
+ } else if !self.path_bitcoindir().is_dir() {
+ println!(
+ "Given --bitcoindir parameter doesn't seem to be a valid directory path.\nRun the program with '-h' for help."
+ );
+ std::process::exit(1);
+ }
+
+ if self.kibodir.is_none() {
+ println!(
+ "You need to set the --kibodir parameter at least once to run the parser.\nRun the program with '-h' for help."
+ );
+ std::process::exit(1);
+ } else if !self.path_kibodir().is_dir() {
+ println!(
+ "Given --kibodir parameter doesn't seem to be a valid directory path.\nRun the program with '-h' for help."
+ );
+ std::process::exit(1);
+ }
+
+ let path = self.path_bitcoindir();
+ if !path.is_dir() {
+ println!("Expect path '{:#?}' to be a directory.", path);
+ std::process::exit(1);
+ }
+
+ if self.to_rpc_auth().is_err() {
+ println!(
+ "No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help."
+ );
+ std::process::exit(1);
+ }
+ }
+
+ fn read(path: &Path) -> Self {
+ fs::read_to_string(path).map_or(Config::default(), |contents| {
+ toml::from_str(&contents).unwrap_or_default()
+ })
+ }
+
+ fn write(&self, path: &Path) -> std::io::Result<()> {
+ fs::write(path, toml::to_string(self).unwrap())
+ }
+
+ pub fn to_rpc_auth(&self) -> color_eyre::Result {
+ let cookie = self.path_cookiefile();
+
+ if cookie.is_file() {
+ Ok(Auth::CookieFile(cookie))
+ } else if self.rpcuser.is_some() && self.rpcpassword.is_some() {
+ Ok(Auth::UserPass(
+ self.rpcuser.clone().unwrap(),
+ self.rpcpassword.clone().unwrap(),
+ ))
+ } else {
+ Err(eyre!("Failed to find correct auth"))
+ }
+ }
+
+ pub fn rpcconnect(&self) -> Option<&String> {
+ self.rpcconnect.as_ref()
+ }
+
+ pub fn rpcport(&self) -> Option {
+ self.rpcport
+ }
+
+ pub fn delay(&self) -> Option {
+ self.delay
+ }
+
+ pub fn dry_run(&self) -> bool {
+ self.dry_run
+ }
+
+ pub fn record_ram_usage(&self) -> bool {
+ self.record_ram_usage
+ }
+
+ pub fn recompute_computed(&self) -> bool {
+ self.recompute_computed
+ }
+
+ pub fn path_bitcoindir(&self) -> PathBuf {
+ Self::fix_user_path(self.bitcoindir.as_ref().unwrap().as_ref())
+ }
+
+ pub fn path_kibodir(&self) -> PathBuf {
+ Self::fix_user_path(self.kibodir.as_ref().unwrap().as_ref())
+ }
+
+ fn path_cookiefile(&self) -> PathBuf {
+ self.rpccookiefile.as_ref().map_or_else(
+ || self.path_bitcoindir().join(".cookie"),
+ |p| Self::fix_user_path(p.as_str()),
+ )
+ }
+
+ fn fix_user_path(path: &str) -> PathBuf {
+ let fix = move |pattern: &str| {
+ if path.starts_with(pattern) {
+ let path = &path.replace(&format!("{pattern}/"), "").replace(pattern, "");
+
+ let home = std::env::var("HOME").unwrap();
+
+ Some(Path::new(&home).join(path))
+ } else {
+ None
+ }
+ };
+
+ fix("~").unwrap_or_else(|| fix("$HOME").unwrap_or_else(|| PathBuf::from(&path)))
+ }
+
+ pub fn path_datasets(&self) -> MapPath {
+ MapPath::from(self.path_kibodir().join(Self::DATASET_DIR_NAME))
+ }
+
+ pub fn path_datasets_last_values(&self) -> MapPath {
+ self.path_datasets().join(&format!("last.{JSON_EXTENSION}"))
+ }
+
+ pub fn path_price(&self) -> MapPath {
+ MapPath::from(self.path_kibodir().join("price"))
+ }
+
+ pub fn path_databases(&self) -> PathBuf {
+ self.path_kibodir().join(Self::DATABASES_DIR_NAME)
+ }
+
+ pub fn path_states(&self) -> PathBuf {
+ self.path_kibodir().join("states")
+ }
+
+ pub fn path_inputs(&self) -> PathBuf {
+ self.path_kibodir().join("inputs")
+ }
+
+ fn path_dot_kibo() -> PathBuf {
+ let home = std::env::var("HOME").unwrap();
+ Path::new(&home).join(".kibo")
+ }
+
+ pub fn path_log() -> PathBuf {
+ Self::path_dot_kibo().join("log")
+ }
+
+ pub fn parser(&self) -> bool {
+ !self.no_parser
+ }
+
+ pub fn server(&self) -> bool {
+ !self.no_server
+ }
+}
diff --git a/parser/src/structs/counter.rs b/_src/structs/counter.rs
similarity index 100%
rename from parser/src/structs/counter.rs
rename to _src/structs/counter.rs
diff --git a/parser/src/structs/date.rs b/_src/structs/date.rs
similarity index 91%
rename from parser/src/structs/date.rs
rename to _src/structs/date.rs
index ec36f4c52..b99e933a4 100644
--- a/parser/src/structs/date.rs
+++ b/_src/structs/date.rs
@@ -44,16 +44,12 @@ impl Date {
Self(chrono::offset::Utc::now().date_naive())
}
- pub fn yesterday() -> Self {
- Self(Self::today().checked_sub_days(Days::new(1)).unwrap())
- }
-
pub fn difference_in_days_between(&self, older: Self) -> u32 {
(**self - *older).num_days() as u32
}
pub fn to_timestamp(self) -> Timestamp {
- Timestamp::wrap(NaiveDateTime::from(*self).and_utc().timestamp() as u32)
+ Timestamp::from(NaiveDateTime::from(*self).and_utc().timestamp() as u32)
}
/// Returns value between 0.0 and 1.0 depending on its completion
@@ -74,17 +70,17 @@ impl Date {
}
}
+ pub fn is_january(&self) -> bool {
+ self.month() == 1
+ }
+
+ pub fn is_july(&self) -> bool {
+ self.month() == 7
+ }
+
pub fn is_first_of_month(&self) -> bool {
self.day() == 1
}
-
- pub fn is_first_of_january(&self) -> bool {
- self.is_first_of_month() && self.month() == 1
- }
-
- pub fn is_first_of_june(&self) -> bool {
- self.is_first_of_month() && self.month() == 6
- }
}
impl MapKey for Date {
diff --git a/parser/src/structs/date_data.rs b/_src/structs/date_data.rs
similarity index 100%
rename from parser/src/structs/date_data.rs
rename to _src/structs/date_data.rs
diff --git a/parser/src/structs/date_map.rs b/_src/structs/date_map.rs
similarity index 92%
rename from parser/src/structs/date_map.rs
rename to _src/structs/date_map.rs
index 76e3c6bde..54b97530c 100644
--- a/parser/src/structs/date_map.rs
+++ b/_src/structs/date_map.rs
@@ -1,10 +1,10 @@
use std::iter::Sum;
-use crate::{Date, HeightMap};
+use super::{
+ AnyMap, Date, DateMapChunkId, GenericMap, Height, HeightMap, MapValue, SerializedDateMap,
+};
-use super::{AnyMap, DateMapChunkId, GenericMap, Height, MapValue, SerializedBTreeMap};
-
-pub type DateMap = GenericMap>;
+pub type DateMap = GenericMap>;
impl DateMap
where
diff --git a/parser/src/structs/date_map_chunk_id.rs b/_src/structs/date_map_chunk_id.rs
similarity index 77%
rename from parser/src/structs/date_map_chunk_id.rs
rename to _src/structs/date_map_chunk_id.rs
index ca3fdac32..07554e24a 100644
--- a/parser/src/structs/date_map_chunk_id.rs
+++ b/_src/structs/date_map_chunk_id.rs
@@ -3,9 +3,7 @@ use std::path::Path;
use allocative::Allocative;
use chrono::Datelike;
-use crate::Date;
-
-use super::MapChunkId;
+use super::{Date, MapChunkId};
#[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Allocative)]
pub struct DateMapChunkId(i32);
@@ -17,7 +15,7 @@ impl DateMapChunkId {
}
impl MapChunkId for DateMapChunkId {
- fn to_name(&self) -> String {
+ fn to_string(&self) -> String {
self.0.to_string()
}
@@ -41,4 +39,12 @@ impl MapChunkId for DateMapChunkId {
fn from_usize(id: usize) -> Self {
Self(id as i32)
}
+
+ fn next(&self) -> Option {
+ self.0.checked_add(1).map(Self)
+ }
+
+ fn previous(&self) -> Option {
+ self.0.checked_sub(1).map(Self)
+ }
}
diff --git a/parser/src/structs/empty_address_data.rs b/_src/structs/empty_address_data.rs
similarity index 94%
rename from parser/src/structs/empty_address_data.rs
rename to _src/structs/empty_address_data.rs
index a7613efcb..48aaa7784 100644
--- a/parser/src/structs/empty_address_data.rs
+++ b/_src/structs/empty_address_data.rs
@@ -1,5 +1,5 @@
use allocative::Allocative;
-use sanakirja::{direct_repr, Storable, UnsizedStorable};
+use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::{AddressData, AddressType, Amount};
diff --git a/parser/src/structs/epoch.rs b/_src/structs/epoch.rs
similarity index 84%
rename from parser/src/structs/epoch.rs
rename to _src/structs/epoch.rs
index ed7108a4c..0bc3aa303 100644
--- a/parser/src/structs/epoch.rs
+++ b/_src/structs/epoch.rs
@@ -9,7 +9,7 @@ impl Epoch {
impl From for Epoch {
fn from(height: Height) -> Self {
- Self(((height.to_usize() / Self::BLOCKS_PER_EPOCH) + 1) as u16)
+ Self::from(&height)
}
}
diff --git a/parser/src/structs/exit.rs b/_src/structs/exit.rs
similarity index 92%
rename from parser/src/structs/exit.rs
rename to _src/structs/exit.rs
index 5122ef361..00408a42e 100644
--- a/parser/src/structs/exit.rs
+++ b/_src/structs/exit.rs
@@ -8,7 +8,7 @@ use std::{
time::Duration,
};
-use super::super::log;
+use log::info;
#[derive(Default, Clone)]
pub struct Exit {
@@ -29,12 +29,12 @@ impl Exit {
let blocked = move || _blocked.load(Ordering::SeqCst);
ctrlc::set_handler(move || {
- log("Exitting...");
+ info!("Exitting...");
active.store(true, Ordering::SeqCst);
if blocked() {
- log("Waiting to exit safely");
+ info!("Waiting to exit safely");
while blocked() {
sleep(Duration::from_millis(50));
diff --git a/parser/src/structs/generic_map.rs b/_src/structs/generic_map.rs
similarity index 91%
rename from parser/src/structs/generic_map.rs
rename to _src/structs/generic_map.rs
index a384f6619..f15c92e05 100644
--- a/parser/src/structs/generic_map.rs
+++ b/_src/structs/generic_map.rs
@@ -1,6 +1,6 @@
use std::{
collections::{BTreeMap, VecDeque},
- fmt::Debug,
+ fmt::{Debug, Display},
fs,
iter::Sum,
mem,
@@ -15,12 +15,11 @@ use ordered_float::OrderedFloat;
use serde::{de::DeserializeOwned, Serialize};
use crate::{
- log,
+ io::Serialization,
utils::{get_percentile, LossyFrom},
- Serialization,
};
-use super::{AnyMap, MapValue};
+use super::{AnyMap, Config, MapPath, MapValue};
#[derive(Debug, Clone, Copy, Allocative, PartialEq, Eq)]
pub enum MapKind {
@@ -30,7 +29,7 @@ pub enum MapKind {
pub trait MapKey
where
- Self: Sized + PartialOrd + Ord + Clone + Copy + Debug,
+ Self: Sized + PartialOrd + Ord + Clone + Copy + Debug + Display,
ChunkId: MapChunkId,
{
fn to_chunk_id(&self) -> ChunkId;
@@ -62,16 +61,21 @@ where
fn get(&self, serialized_key: &Key) -> Option<&Value>;
fn last(&self) -> Option<&Value>;
fn extend(&mut self, map: BTreeMap);
+ fn import_all(path: &Path, serialization: &Serialization) -> Self;
+ fn to_csv(self, id: &str) -> String;
+ fn map(&self) -> &impl Serialize;
}
pub trait MapChunkId
where
Self: Ord + Debug + Copy + Clone,
{
- fn to_name(&self) -> String;
+ fn to_string(&self) -> String;
fn from_path(path: &Path) -> color_eyre::Result;
fn to_usize(self) -> usize;
fn from_usize(id: usize) -> Self;
+ fn previous(&self) -> Option;
+ fn next(&self) -> Option;
}
#[derive(Debug, Allocative)]
@@ -79,8 +83,9 @@ pub struct GenericMap {
version: u32,
kind: MapKind,
- path_all: PathBuf,
- path_last: Option,
+ path_all: MapPath,
+ path_parent: MapPath,
+ path_last: Option,
chunks_in_memory: usize,
@@ -100,15 +105,15 @@ where
Key: MapKey,
Serialized: MapSerialized,
{
- pub fn new_bin(version: u32, kind: MapKind, path: &str) -> Self {
+ pub fn new_bin(version: u32, kind: MapKind, path: &MapPath) -> Self {
Self::new(version, kind, path, Serialization::Binary, 1, true)
}
- pub fn _new_bin(version: u32, kind: MapKind, path: &str, export_last: bool) -> Self {
+ pub fn _new_bin(version: u32, kind: MapKind, path: &MapPath, export_last: bool) -> Self {
Self::new(version, kind, path, Serialization::Binary, 1, export_last)
}
- pub fn new_json(version: u32, kind: MapKind, path: &str, export_last: bool) -> Self {
+ pub fn new_json(version: u32, kind: MapKind, path: &MapPath, export_last: bool) -> Self {
Self::new(
version,
kind,
@@ -122,7 +127,7 @@ where
fn new(
version: u32,
kind: MapKind,
- path: &str,
+ path: &MapPath,
serialization: Serialization,
chunks_in_memory: usize,
export_last: bool,
@@ -131,15 +136,16 @@ where
panic!("Should always have at least the latest chunk in memory");
}
- let path = path.replace(['-', '_', ' '], "/");
+ let path_all = path.join(Key::map_name());
- let path_all = PathBuf::from(format!("{path}/{}", Key::map_name()));
-
- fs::create_dir_all(&path_all).unwrap();
+ fs::create_dir_all(&*path_all).unwrap_or_else(|_| {
+ dbg!(&path_all);
+ panic!()
+ });
let path_last = {
if export_last {
- Some(PathBuf::from(format!("{path}/last")))
+ Some(path.join("last"))
} else {
None
}
@@ -150,6 +156,7 @@ where
kind,
path_all,
+ path_parent: path.to_owned(),
path_last,
chunks_in_memory,
@@ -177,21 +184,25 @@ where
}
});
- s.initial_last_key = s
+ s.set_initial_keys();
+
+ // if s.initial_first_unsafe_key.is_none() {
+ // log(&format!("Missing dataset: {path:?}/{}", Key::map_name()));
+ // }
+
+ s
+ }
+
+ fn set_initial_keys(&mut self) {
+ self.initial_last_key = self
.imported
.iter()
.last()
.and_then(|(last_chunk_id, serialized)| serialized.get_last_key(last_chunk_id));
- s.initial_first_unsafe_key = s
+ self.initial_first_unsafe_key = self
.initial_last_key
.and_then(|last_key| last_key.to_first_unsafe());
-
- if s.initial_first_unsafe_key.is_none() {
- log(&format!("Missing dataset: {path}/{}", Key::map_name()));
- }
-
- s
}
fn read_dir(&self) -> BTreeMap {
@@ -308,11 +319,27 @@ where
Key: MapKey,
Serialized: MapSerialized,
{
+ fn id(&self, config: &Config) -> String {
+ let path_to_string = |p: &Path| p.to_str().unwrap().to_owned();
+ path_to_string(self.path_parent())
+ .replace(&format!("{}/", path_to_string(&config.path_kibodir())), "")
+ .replace(&format!("{}/", Config::DATASET_DIR_NAME), "")
+ .replace("/", "-")
+ }
+
+ fn serialization(&self) -> Serialization {
+ self.serialization
+ }
+
fn path(&self) -> &Path {
&self.path_all
}
- fn path_last(&self) -> &Option {
+ fn path_parent(&self) -> &Path {
+ &self.path_parent
+ }
+
+ fn path_last(&self) -> &Option {
&self.path_last
}
@@ -323,10 +350,14 @@ where
.and_then(|v| serde_json::to_value(v).ok())
}
- fn t_name(&self) -> &str {
+ fn type_name(&self) -> &str {
std::any::type_name::()
}
+ fn key_name(&self) -> &str {
+ Key::map_name()
+ }
+
fn pre_export(&mut self) {
self.to_insert.iter_mut().for_each(|(chunk_id, map)| {
if let Some((key, _)) = map.first_key_value() {
@@ -351,6 +382,8 @@ where
.or_insert(Serialized::new(self.version))
.extend(mem::take(map));
});
+
+ self.set_initial_keys();
}
fn export(&self) -> color_eyre::Result<()> {
@@ -367,15 +400,15 @@ where
panic!();
});
- let path = self.path_all.join(chunk_id.to_name());
+ let path = self.path_all.join(&chunk_id.to_string());
- self.serialization.export(Path::new(&path), serialized)?;
+ self.serialization.export(&path, serialized)?;
// Export last
if index == len - 1 {
if let Some(path_last) = self.path_last.as_ref() {
self.serialization
- .export(Path::new(path_last), serialized.last().unwrap())?;
+ .export(path_last, serialized.last().unwrap())?;
}
}
@@ -418,22 +451,22 @@ where
Key: MapKey,
Serialized: MapSerialized,
{
- pub fn sum_keys(&mut self, keys: &[Key]) -> Value
- where
- Value: Sum,
- {
- keys.iter()
- .map(|key| self.get_or_import(key).unwrap())
- .sum::()
- }
+ // pub fn sum_keys(&mut self, keys: &[Key]) -> Value
+ // where
+ // Value: Sum,
+ // {
+ // keys.iter()
+ // .map(|key| self.get_or_import(key).unwrap())
+ // .sum::()
+ // }
- pub fn average_keys(&mut self, keys: &[Key]) -> f32
- where
- Value: Sum,
- f32: LossyFrom,
- {
- f32::lossy_from(self.sum_keys(keys)) / keys.len() as f32
- }
+ // pub fn average_keys(&mut self, keys: &[Key]) -> f32
+ // where
+ // Value: Sum,
+ // f32: LossyFrom,
+ // {
+ // f32::lossy_from(self.sum_keys(keys)) / keys.len() as f32
+ // }
pub fn multi_insert(&mut self, keys: &[Key], mut callback: F)
where
diff --git a/parser/src/structs/height.rs b/_src/structs/height.rs
similarity index 90%
rename from parser/src/structs/height.rs
rename to _src/structs/height.rs
index b3d811eff..c788e6afb 100644
--- a/parser/src/structs/height.rs
+++ b/_src/structs/height.rs
@@ -5,13 +5,11 @@ use std::{
use allocative::Allocative;
use bincode::{Decode, Encode};
-use biter::NUMBER_OF_UNSAFE_BLOCKS;
+use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
-use crate::HEIGHT_MAP_CHUNK_SIZE;
-
-use super::{HeightMapChunkId, MapKey};
+use super::{HEIGHT_MAP_CHUNK_SIZE, HeightMapChunkId, MapKey};
#[derive(
Debug,
@@ -47,11 +45,11 @@ impl Height {
**self < (block_count - NUMBER_OF_UNSAFE_BLOCKS) as u32
}
- pub fn iter_range_inclusive(first: Height, last: Height) -> impl Iterator- {
- let range = (*first)..=(*last);
+ // pub fn iter_range_inclusive(first: Height, last: Height) -> impl Iterator
- {
+ // let range = (*first)..=(*last);
- range.into_iter().map(Height::new)
- }
+ // range.into_iter().map(Height::new)
+ // }
}
impl PartialEq for Height {
diff --git a/parser/src/structs/height_map.rs b/_src/structs/height_map.rs
similarity index 97%
rename from parser/src/structs/height_map.rs
rename to _src/structs/height_map.rs
index bd2f36a73..f768f96b0 100644
--- a/parser/src/structs/height_map.rs
+++ b/_src/structs/height_map.rs
@@ -2,9 +2,7 @@ use std::{iter::Sum, ops::RangeInclusive};
use itertools::Itertools;
-use crate::SerializedVec;
-
-use super::{AnyMap, GenericMap, Height, HeightMapChunkId, MapValue};
+use super::{AnyMap, GenericMap, Height, HeightMapChunkId, MapValue, SerializedVec};
pub const HEIGHT_MAP_CHUNK_SIZE: u32 = 10_000;
diff --git a/parser/src/structs/height_map_chunk_id.rs b/_src/structs/height_map_chunk_id.rs
similarity index 74%
rename from parser/src/structs/height_map_chunk_id.rs
rename to _src/structs/height_map_chunk_id.rs
index 6f662529d..bfe82058a 100644
--- a/parser/src/structs/height_map_chunk_id.rs
+++ b/_src/structs/height_map_chunk_id.rs
@@ -3,9 +3,7 @@ use std::path::Path;
use allocative::Allocative;
use derive_deref::{Deref, DerefMut};
-use crate::HEIGHT_MAP_CHUNK_SIZE;
-
-use super::{Height, MapChunkId};
+use super::{Height, MapChunkId, HEIGHT_MAP_CHUNK_SIZE};
#[derive(
Debug, Default, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Allocative, Deref, DerefMut,
@@ -21,7 +19,7 @@ impl HeightMapChunkId {
}
impl MapChunkId for HeightMapChunkId {
- fn to_name(&self) -> String {
+ fn to_string(&self) -> String {
let start = ***self;
let end = start + HEIGHT_MAP_CHUNK_SIZE;
@@ -48,4 +46,14 @@ impl MapChunkId for HeightMapChunkId {
fn from_usize(id: usize) -> Self {
Self(Height::new(id as u32))
}
+
+ fn next(&self) -> Option {
+ self.checked_add(HEIGHT_MAP_CHUNK_SIZE)
+ .map(|h| Self(Height::new(h)))
+ }
+
+ fn previous(&self) -> Option {
+ self.checked_sub(HEIGHT_MAP_CHUNK_SIZE)
+ .map(|h| Self(Height::new(h)))
+ }
}
diff --git a/_src/structs/instant.rs b/_src/structs/instant.rs
new file mode 100644
index 000000000..9495f71eb
--- /dev/null
+++ b/_src/structs/instant.rs
@@ -0,0 +1,15 @@
+use std::time::Instant;
+
+use color_eyre::owo_colors::OwoColorize;
+
+pub trait DisplayInstant {
+ fn display(&self) -> String;
+}
+
+impl DisplayInstant for Instant {
+ fn display(&self) -> String {
+ format!("{:.2}s", self.elapsed().as_secs_f32())
+ .bright_black()
+ .to_string()
+ }
+}
diff --git a/parser/src/structs/liquidity.rs b/_src/structs/liquidity.rs
similarity index 100%
rename from parser/src/structs/liquidity.rs
rename to _src/structs/liquidity.rs
diff --git a/_src/structs/map_path.rs b/_src/structs/map_path.rs
new file mode 100644
index 000000000..ebf056374
--- /dev/null
+++ b/_src/structs/map_path.rs
@@ -0,0 +1,24 @@
+use std::path::PathBuf;
+
+use allocative::Allocative;
+use derive_deref::{Deref, DerefMut};
+
+#[derive(Debug, Clone, Deref, DerefMut, Allocative)]
+pub struct MapPath(PathBuf);
+
+impl MapPath {
+ pub fn join(&self, path: &str) -> Self {
+ let path = path.replace(['-', '_', ' '], "/");
+ Self(self.0.join(path))
+ }
+
+ pub fn unwrap(&self) -> &PathBuf {
+ &self.0
+ }
+}
+
+impl From for MapPath {
+ fn from(value: PathBuf) -> Self {
+ Self(value)
+ }
+}
diff --git a/parser/src/structs/map_value.rs b/_src/structs/map_value.rs
similarity index 93%
rename from parser/src/structs/map_value.rs
rename to _src/structs/map_value.rs
index f9790c72d..4f1dcf9fa 100644
--- a/parser/src/structs/map_value.rs
+++ b/_src/structs/map_value.rs
@@ -1,4 +1,4 @@
-use std::fmt::Debug;
+use std::fmt::{Debug, Display};
use allocative::Allocative;
use bincode::{Decode, Encode};
@@ -18,6 +18,7 @@ pub trait MapValue:
+ Sync
+ Send
+ Allocative
+ + Display
{
}
diff --git a/parser/src/structs/mod.rs b/_src/structs/mod.rs
similarity index 94%
rename from parser/src/structs/mod.rs
rename to _src/structs/mod.rs
index 1278f5844..f1f012319 100644
--- a/parser/src/structs/mod.rs
+++ b/_src/structs/mod.rs
@@ -24,11 +24,14 @@ mod generic_map;
mod height;
mod height_map;
mod height_map_chunk_id;
+mod instant;
mod liquidity;
+mod map_path;
mod map_value;
mod ohlc;
mod partial_txout_data;
mod price;
+mod rpc;
mod sent_data;
mod serialized_btreemap;
mod serialized_vec;
@@ -62,7 +65,9 @@ pub use generic_map::*;
pub use height::*;
pub use height_map::*;
pub use height_map_chunk_id::*;
+pub use instant::*;
pub use liquidity::*;
+pub use map_path::*;
pub use map_value::*;
pub use ohlc::*;
pub use partial_txout_data::*;
diff --git a/parser/src/structs/ohlc.rs b/_src/structs/ohlc.rs
similarity index 53%
rename from parser/src/structs/ohlc.rs
rename to _src/structs/ohlc.rs
index 666a52b1f..16fc833a5 100644
--- a/parser/src/structs/ohlc.rs
+++ b/_src/structs/ohlc.rs
@@ -1,3 +1,5 @@
+use std::fmt::{self};
+
use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
@@ -10,3 +12,13 @@ pub struct OHLC {
pub low: f32,
pub close: f32,
}
+
+impl fmt::Display for OHLC {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(
+ f,
+ "{{ open: {}, high: {}, low: {}, close: {} }}",
+ self.open, self.high, self.low, self.close
+ )
+ }
+}
diff --git a/parser/src/structs/partial_txout_data.rs b/_src/structs/partial_txout_data.rs
similarity index 100%
rename from parser/src/structs/partial_txout_data.rs
rename to _src/structs/partial_txout_data.rs
diff --git a/parser/src/structs/price.rs b/_src/structs/price.rs
similarity index 100%
rename from parser/src/structs/price.rs
rename to _src/structs/price.rs
diff --git a/parser/src/structs/ram.rs b/_src/structs/ram.rs
similarity index 100%
rename from parser/src/structs/ram.rs
rename to _src/structs/ram.rs
diff --git a/_src/structs/rpc.rs b/_src/structs/rpc.rs
new file mode 100644
index 000000000..885c6bb71
--- /dev/null
+++ b/_src/structs/rpc.rs
@@ -0,0 +1,17 @@
+use brk_parser::bitcoincore_rpc::Client;
+
+use crate::structs::Config;
+
+impl From<&Config> for Client {
+ fn from(config: &Config) -> Self {
+ Client::new(
+ &format!(
+ "http://{}:{}",
+ config.rpcconnect().unwrap_or(&"localhost".to_owned()),
+ config.rpcport().unwrap_or(8332)
+ ),
+ config.to_rpc_auth().unwrap(),
+ )
+ .unwrap()
+ }
+}
diff --git a/parser/src/structs/sent_data.rs b/_src/structs/sent_data.rs
similarity index 100%
rename from parser/src/structs/sent_data.rs
rename to _src/structs/sent_data.rs
diff --git a/parser/src/structs/serialized_btreemap.rs b/_src/structs/serialized_btreemap.rs
similarity index 72%
rename from parser/src/structs/serialized_btreemap.rs
rename to _src/structs/serialized_btreemap.rs
index 2f1723ff1..3b5b730d4 100644
--- a/parser/src/structs/serialized_btreemap.rs
+++ b/_src/structs/serialized_btreemap.rs
@@ -4,9 +4,12 @@ use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
-use crate::Serialization;
+use crate::io::Serialization;
-use super::{DateMap, MapChunkId, MapKey, MapSerialized, MapValue};
+use super::{Date, DateMap, MapChunkId, MapKey, MapSerialized, MapValue, Timestamp};
+
+pub type SerializedDateMap = SerializedBTreeMap;
+pub type SerializedTimeMap = SerializedBTreeMap;
#[derive(Debug, Default, Serialize, Deserialize, Encode, Decode, Allocative)]
pub struct SerializedBTreeMap
@@ -17,41 +20,11 @@ where
pub map: BTreeMap,
}
-impl SerializedBTreeMap
-where
- Key: Ord,
-{
- pub fn import_all(path: &Path, serialization: &Serialization) -> Self
- where
- Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
- ChunkId: MapChunkId,
- Key: MapKey,
- Value: MapValue,
- {
- let mut s = None;
-
- DateMap::::_read_dir(path, serialization)
- .iter()
- .for_each(|(_, path)| {
- let map = serialization.import::(path).unwrap();
-
- if s.is_none() {
- s.replace(map);
- } else {
- #[allow(clippy::unnecessary_unwrap)]
- s.as_mut().unwrap().map.extend(map.map);
- }
- });
-
- s.unwrap()
- }
-}
-
impl MapSerialized for SerializedBTreeMap
where
Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
ChunkId: MapChunkId,
- Key: MapKey,
+ Key: MapKey + Serialize,
Value: MapValue,
{
fn new(version: u32) -> Self {
@@ -80,4 +53,35 @@ where
fn extend(&mut self, map: BTreeMap) {
self.map.extend(map)
}
+
+ fn import_all(path: &Path, serialization: &Serialization) -> Self {
+ let mut s = None;
+
+ DateMap::::_read_dir(path, serialization)
+ .iter()
+ .for_each(|(_, path)| {
+ let map = serialization.import::(path).unwrap();
+
+ if s.is_none() {
+ s.replace(map);
+ } else {
+ #[allow(clippy::unnecessary_unwrap)]
+ s.as_mut().unwrap().map.extend(map.map);
+ }
+ });
+
+ s.unwrap()
+ }
+
+ fn to_csv(self, id: &str) -> String {
+ let mut csv = format!("{},{}\n", Key::map_name(), id);
+ self.map.iter().for_each(|(k, v)| {
+ csv += &format!("{},{}\n", k, v);
+ });
+ csv
+ }
+
+ fn map(&self) -> &impl Serialize {
+ &self.map
+ }
}
diff --git a/parser/src/structs/serialized_vec.rs b/_src/structs/serialized_vec.rs
similarity index 82%
rename from parser/src/structs/serialized_vec.rs
rename to _src/structs/serialized_vec.rs
index 3d61a9738..73efd3f3e 100644
--- a/parser/src/structs/serialized_vec.rs
+++ b/_src/structs/serialized_vec.rs
@@ -4,7 +4,7 @@ use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
-use crate::Serialization;
+use crate::io::Serialization;
use super::{HeightMap, MapChunkId, MapKey, MapSerialized, MapValue};
@@ -15,27 +15,8 @@ pub struct SerializedVec {
}
impl SerializedVec {
- pub fn import_all(path: &Path, serialization: &Serialization) -> Self
- where
- Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
- Value: MapValue,
- {
- let mut s = None;
-
- HeightMap::::_read_dir(path, serialization)
- .iter()
- .for_each(|(_, path)| {
- let mut map = serialization.import::(path).unwrap();
-
- if s.is_none() {
- s.replace(map);
- } else {
- #[allow(clippy::unnecessary_unwrap)]
- s.as_mut().unwrap().map.append(&mut map.map);
- }
- });
-
- s.unwrap()
+ pub fn get_index(&self, index: usize) -> Option<&Value> {
+ self.map.get(index)
}
}
@@ -87,4 +68,39 @@ where
}
});
}
+
+ fn import_all(path: &Path, serialization: &Serialization) -> Self
+ where
+ Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
+ Value: MapValue,
+ {
+ let mut s = None;
+
+ HeightMap::::_read_dir(path, serialization)
+ .iter()
+ .for_each(|(_, path)| {
+ let mut map = serialization.import::(path).unwrap();
+
+ if s.is_none() {
+ s.replace(map);
+ } else {
+ #[allow(clippy::unnecessary_unwrap)]
+ s.as_mut().unwrap().map.append(&mut map.map);
+ }
+ });
+
+ s.unwrap()
+ }
+
+ fn to_csv(self, id: &str) -> String {
+ let mut csv = format!("{},{}\n", Key::map_name(), id);
+ self.map.iter().enumerate().for_each(|(k, v)| {
+ csv += &format!("{:?},{:?}\n", k, v);
+ });
+ csv
+ }
+
+ fn map(&self) -> &impl Serialize {
+ &self.map
+ }
}
diff --git a/parser/src/structs/timestamp.rs b/_src/structs/timestamp.rs
similarity index 52%
rename from parser/src/structs/timestamp.rs
rename to _src/structs/timestamp.rs
index 04ce1b1ad..0d8cc88df 100644
--- a/parser/src/structs/timestamp.rs
+++ b/_src/structs/timestamp.rs
@@ -1,14 +1,14 @@
-use std::ops::Sub;
+use std::{fmt, ops::Sub};
use allocative::Allocative;
use bincode::{Decode, Encode};
-use chrono::{Datelike, NaiveDateTime, NaiveTime, TimeZone, Timelike, Utc};
+use chrono::{NaiveDateTime, NaiveTime, TimeZone, Timelike, Utc};
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use crate::utils::{ONE_DAY_IN_S, ONE_HOUR_IN_S};
-use super::Date;
+use super::{Date, HeightMapChunkId, MapKey};
#[derive(
Debug,
@@ -32,10 +32,6 @@ pub struct Timestamp(u32);
impl Timestamp {
pub const ZERO: Self = Self(0);
- pub fn wrap(timestamp: u32) -> Self {
- Self(timestamp)
- }
-
pub fn now() -> Self {
Self(chrono::offset::Utc::now().timestamp() as u32)
}
@@ -48,14 +44,10 @@ impl Timestamp {
)
}
- pub fn to_year(self) -> u32 {
- self.to_date().year() as u32
- }
-
pub fn to_floored_seconds(self) -> Self {
let date_time = Utc.timestamp_opt(i64::from(self.0), 0).unwrap();
- Self::wrap(
+ Self::from(
NaiveDateTime::new(
date_time.date_naive(),
NaiveTime::from_hms_opt(date_time.hour(), date_time.minute(), 0).unwrap(),
@@ -74,7 +66,7 @@ impl Timestamp {
}
pub fn older_by_1h_plus_than(&self, younger: Self) -> bool {
- younger.checked_sub(**self).unwrap_or_default() > ONE_HOUR_IN_S as u32
+ (*younger).checked_sub(**self).unwrap_or_default() > ONE_HOUR_IN_S as u32
}
}
@@ -82,6 +74,64 @@ impl Sub for Timestamp {
type Output = Self;
fn sub(self, rhs: Self) -> Self::Output {
- Self::wrap(self.0 - rhs.0)
+ Self::from(self.0 - rhs.0)
+ }
+}
+
+impl fmt::Display for Timestamp {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "{}", **self)
+ }
+}
+
+impl From for Timestamp {
+ fn from(value: u32) -> Self {
+ Self(value)
+ }
+}
+
+impl MapKey for Timestamp {
+ fn to_chunk_id(&self) -> HeightMapChunkId {
+ unreachable!();
+ }
+
+ fn to_first_unsafe(&self) -> Option {
+ unreachable!();
+ }
+
+ fn to_serialized_key(&self) -> Self {
+ unreachable!();
+ }
+
+ fn is_out_of_bounds(&self) -> bool {
+ unreachable!();
+ }
+
+ fn is_first(&self) -> bool {
+ unreachable!();
+ }
+
+ fn checked_sub(&self, _: usize) -> Option {
+ unreachable!();
+ }
+
+ fn min_percentile_key() -> Self {
+ unreachable!();
+ }
+
+ fn iter_up_to(&self, other: &Self) -> impl Iterator
- {
+ (**self..=**other).map(Timestamp::from)
+ }
+
+ fn map_name<'a>() -> &'a str {
+ "timestamp"
+ }
+
+ fn to_usize(&self) -> usize {
+ (**self) as usize
+ }
+
+ fn from_usize(t: usize) -> Self {
+ Self(t as u32)
}
}
diff --git a/parser/src/structs/tx_data.rs b/_src/structs/tx_data.rs
similarity index 90%
rename from parser/src/structs/tx_data.rs
rename to _src/structs/tx_data.rs
index 1a577e8be..162ed456f 100644
--- a/parser/src/structs/tx_data.rs
+++ b/_src/structs/tx_data.rs
@@ -1,5 +1,5 @@
use allocative::Allocative;
-use sanakirja::{direct_repr, Storable, UnsizedStorable};
+use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::BlockPath;
diff --git a/parser/src/structs/txout_index.rs b/_src/structs/txout_index.rs
similarity index 92%
rename from parser/src/structs/txout_index.rs
rename to _src/structs/txout_index.rs
index 6da7f588d..ebb36775f 100644
--- a/parser/src/structs/txout_index.rs
+++ b/_src/structs/txout_index.rs
@@ -1,6 +1,6 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
-use sanakirja::{direct_repr, Storable, UnsizedStorable};
+use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Encode, Decode, Allocative)]
pub struct TxoutIndex {
diff --git a/parser/src/utils/consts.rs b/_src/utils/consts.rs
similarity index 100%
rename from parser/src/utils/consts.rs
rename to _src/utils/consts.rs
diff --git a/parser/src/utils/flamegraph.rs b/_src/utils/flamegraph.rs
similarity index 84%
rename from parser/src/utils/flamegraph.rs
rename to _src/utils/flamegraph.rs
index 2a40ab398..2e96ce6e3 100644
--- a/parser/src/utils/flamegraph.rs
+++ b/_src/utils/flamegraph.rs
@@ -2,10 +2,15 @@ use std::{fs, path::PathBuf};
use chrono::Local;
-use crate::{databases::Databases, datasets::AllDatasets, states::States, structs::Height};
+use crate::{
+ parser::{Databases, Datasets, States},
+ structs::Height,
+};
+
+// use crate::{databases::Databases, datasets::AllDatasets, states::States, structs::Height};
pub fn generate_allocation_files(
- datasets: &AllDatasets,
+ datasets: &Datasets,
databases: &Databases,
states: &States,
last_height: Height,
diff --git a/_src/utils/log.rs b/_src/utils/log.rs
new file mode 100644
index 000000000..4679d3339
--- /dev/null
+++ b/_src/utils/log.rs
@@ -0,0 +1,82 @@
+use std::{
+ fmt::Display,
+ fs::{self, OpenOptions},
+ io::Write,
+};
+
+use chrono::Local;
+use color_eyre::owo_colors::OwoColorize;
+use env_logger::Env;
+
+use crate::structs::Config;
+
+#[inline(always)]
+pub fn init_log() {
+ let _ = fs::remove_file(Config::path_log());
+
+ let file = Box::new(
+ OpenOptions::new()
+ .create(true)
+ .append(true)
+ .open(Config::path_log())
+ .unwrap(),
+ );
+
+ env_logger::Builder::from_env(
+ Env::default().default_filter_or(format!("{}=info", env!("CARGO_PKG_NAME"))),
+ )
+ .format(move |buf, record| {
+ let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S"));
+ let level = record.level().as_str().to_lowercase();
+ let level = format!("{:5}", level);
+ let target = record.target();
+ let dash = "-";
+ let args = record.args();
+
+ let _ = write(
+ file.try_clone().unwrap(),
+ &date_time,
+ target,
+ &level,
+ dash,
+ args,
+ );
+
+ let colored_date_time = date_time.bright_black();
+ let colored_level = match level.chars().next().unwrap() {
+ 'e' => level.red().to_string(),
+ 'w' => level.yellow().to_string(),
+ 'i' => level.green().to_string(),
+ 'd' => level.blue().to_string(),
+ 't' => level.cyan().to_string(),
+ _ => panic!(),
+ };
+ let colored_dash = dash.bright_black();
+
+ write(
+ buf,
+ colored_date_time,
+ target,
+ colored_level,
+ colored_dash,
+ args,
+ )
+ })
+ .init();
+}
+
+fn write(
+ mut buf: impl Write,
+ date_time: impl Display,
+ _target: impl Display,
+ level: impl Display,
+ dash: impl Display,
+ args: impl Display,
+) -> Result<(), std::io::Error> {
+ writeln!(buf, "{} {} {} {}", date_time, dash, level, args)
+ // writeln!(
+ // buf,
+ // "{} {} {} {} {}",
+ // date_time, _target, level, dash, args
+ // )
+}
diff --git a/parser/src/utils/lossy.rs b/_src/utils/lossy.rs
similarity index 100%
rename from parser/src/utils/lossy.rs
rename to _src/utils/lossy.rs
diff --git a/parser/src/utils/mod.rs b/_src/utils/mod.rs
similarity index 89%
rename from parser/src/utils/mod.rs
rename to _src/utils/mod.rs
index 18c6f8d80..59742bfe6 100644
--- a/parser/src/utils/mod.rs
+++ b/_src/utils/mod.rs
@@ -4,7 +4,6 @@ mod log;
mod lossy;
mod percentile;
mod retry;
-mod rpc;
mod time;
pub use consts::*;
@@ -13,5 +12,4 @@ pub use log::*;
pub use lossy::*;
pub use percentile::*;
pub use retry::*;
-pub use rpc::*;
pub use time::*;
diff --git a/parser/src/utils/percentile.rs b/_src/utils/percentile.rs
similarity index 100%
rename from parser/src/utils/percentile.rs
rename to _src/utils/percentile.rs
diff --git a/parser/src/utils/retry.rs b/_src/utils/retry.rs
similarity index 89%
rename from parser/src/utils/retry.rs
rename to _src/utils/retry.rs
index 6f22e968c..f200ced47 100644
--- a/parser/src/utils/retry.rs
+++ b/_src/utils/retry.rs
@@ -5,10 +5,6 @@ pub fn retry(
sleep_in_s: u64,
retries: usize,
) -> color_eyre::Result {
- if retries < 1 {
- unreachable!()
- }
-
let mut i = 0;
loop {
diff --git a/_src/utils/time.rs b/_src/utils/time.rs
new file mode 100644
index 000000000..2c6df6103
--- /dev/null
+++ b/_src/utils/time.rs
@@ -0,0 +1,18 @@
+use std::time::Instant;
+
+use log::info;
+
+use crate::structs::DisplayInstant;
+
+pub fn time(text: &str, function: F) -> T
+where
+ F: FnOnce() -> T,
+{
+ let time = Instant::now();
+
+ let returned = function();
+
+ info!("{text} {}", time.display());
+
+ returned
+}
diff --git a/biter/Cargo.toml b/biter/Cargo.toml
deleted file mode 100644
index f01e3608d..000000000
--- a/biter/Cargo.toml
+++ /dev/null
@@ -1,19 +0,0 @@
-[package]
-name = "biter"
-description = "A very fast Bitcoin block iterator"
-version = "0.1.1"
-license = "MIT"
-repository = "https://github.com/kibo-money/kibo/tree/main/biter"
-keywords = ["bitcoin", "block", "iterator"]
-categories = ["cryptography::cryptocurrencies", "encoding"]
-edition = "2021"
-
-[dependencies]
-bitcoin = { version = "0.32.2", features = ["serde"] }
-rayon = "1.10.0"
-crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = "1.0.122"
-derived-deref = "2.1.0"
-bitcoincore-rpc = "0.19.0"
-# tokio = { version = "1.39.2", features = ["rt-multi-thread"] }
diff --git a/biter/README.md b/biter/README.md
deleted file mode 100644
index 932108dac..000000000
--- a/biter/README.md
+++ /dev/null
@@ -1,62 +0,0 @@
-# Biter
-
-Biter (Bitcoin Block Iterator) is a very fast and simple Rust library which reads raw block files (*blkXXXXX.dat*) from Bitcoin Core Node and creates an iterator over all the requested blocks in sequential order (0, 1, 2, ...).
-
-The element returned by the iterator is a tuple which includes the:
-- Height: `usize`
-- Block: `Block` (from `bitcoin-rust`)
-- Block's Hash: `BlockHash` (also from `bitcoin-rust`)
-
-## Example
-
-```rust
-use bitcoincore_rpc::{Auth, Client};
-
-fn main() {
- let i = std::time::Instant::now();
-
- // Path to the Bitcoin data directory
- let data_dir = "../../bitcoin";
-
- // Path to the export directory where a mini blk indexer will be exported
- let export_dir = "./target";
-
- // Inclusive starting height of the blocks received, `None` for 0
- let start = Some(850_000);
-
- // Inclusive ending height of the blocks received, `None` for the last one
- let end = None;
-
- // RPC client to filter out forks
- let url = "http://localhost:8332";
- let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
- let rpc = Client::new(url, auth).unwrap();
-
- // Create channel receiver then iterate over the blocks
- biter::new(data_dir, export_dir, start, end, rpc)
- .iter()
- .for_each(|(height, _block, hash)| {
- println!("{height}: {hash}");
- });
-
- dbg!(i.elapsed());
-}
-
-```
-
-## Requirements
-
-Even though it reads *blkXXXXX.dat* files, it **needs** `bitcoind` to run with the RPC server to filter out block forks.
-
-Peak memory should be around 500MB.
-
-## Comparaison
-
-| | [biter](https://crates.io/crates/biter) | [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) | [blocks_iterator](https://crates.io/crates/blocks_iterator) |
-| --- | --- | --- | --- |
-| Run **with** `bitcoind` | Yes ✅ | No ❌ | Yes ✅ |
-| Run **without** `bitcoind` | No ❌ | Yes ✅ | Yes ✅ |
-| `0..=855_000` | 16mn40s | 17mn 46s | > 2h |
-| `800_000..=855_000` | 2mn 53s (16mn40s if first run) | 3mn 2s | > 2h |
-
-*Benchmarked on a Macbook Pro M3 Pro*
diff --git a/biter/src/blk_index_to_blk_recap.rs b/biter/src/blk_index_to_blk_recap.rs
deleted file mode 100644
index 9c9d53c53..000000000
--- a/biter/src/blk_index_to_blk_recap.rs
+++ /dev/null
@@ -1,105 +0,0 @@
-use std::{
- cmp::Ordering,
- collections::BTreeMap,
- fs::{self, File},
- io::{BufReader, BufWriter},
- path::PathBuf,
-};
-
-use derived_deref::{Deref, DerefMut};
-
-use crate::{blk_recap::BlkRecap, BlkMetadataAndBlock};
-
-#[derive(Deref, DerefMut, Debug)]
-pub struct BlkIndexToBlkRecap {
- path: String,
- #[target]
- tree: BTreeMap,
-}
-
-impl BlkIndexToBlkRecap {
- pub fn import(blocks_dir: &BTreeMap, export_dir: &str) -> Self {
- let path = format!("{export_dir}/blk_index_to_blk_recap.json");
-
- let tree = {
- fs::create_dir_all(export_dir).unwrap();
-
- if let Ok(file) = File::open(&path) {
- let reader = BufReader::new(file);
- serde_json::from_reader(reader).unwrap_or_default()
- } else {
- BTreeMap::default()
- }
- };
-
- let mut this = Self { path, tree };
-
- this.clean_outdated(blocks_dir);
-
- this
- }
-
- pub fn clean_outdated(&mut self, blocks_dir: &BTreeMap) {
- blocks_dir.iter().for_each(|(blk_index, blk_path)| {
- if let Some(blk_recap) = self.get(blk_index) {
- if blk_recap.has_different_modified_time(blk_path) {
- self.remove(blk_index);
- }
- }
- });
- }
-
- pub fn get_start_recap(&self, start: Option) -> Option<(usize, BlkRecap)> {
- if let Some(start) = start {
- let (last_key, last_value) = self.last_key_value()?;
-
- if last_value.height() < start {
- return Some((*last_key, *last_value));
- } else if let Some((blk_index, _)) = self
- .iter()
- .find(|(_, blk_recap)| blk_recap.is_younger_than(start))
- {
- if *blk_index != 0 {
- let blk_index = *blk_index - 1;
- return Some((blk_index, *self.get(&blk_index).unwrap()));
- }
- }
- }
-
- None
- }
-
- pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: usize) {
- let blk_index = blk_metadata_and_block.blk_metadata.index;
-
- if let Some(last_entry) = self.last_entry() {
- // if last_entry.get().is_older_than(height) {
- match last_entry.key().cmp(&blk_index) {
- Ordering::Greater => {
- last_entry.remove_entry();
- }
- Ordering::Less => {
- self.insert(blk_index, BlkRecap::from(height, blk_metadata_and_block));
- }
- Ordering::Equal => {}
- };
- // }
- } else {
- if blk_index != 0 || height != 0 {
- // dbg!(blk_index, height);
- unreachable!();
- }
-
- self.insert(blk_index, BlkRecap::first(blk_metadata_and_block));
- }
- }
-
- pub fn export(&self) {
- let file = File::create(&self.path).unwrap_or_else(|_| {
- dbg!(&self.path);
- panic!("No such file or directory")
- });
-
- serde_json::to_writer_pretty(&mut BufWriter::new(file), &self.tree).unwrap();
- }
-}
diff --git a/biter/src/blk_metadata_and_block.rs b/biter/src/blk_metadata_and_block.rs
deleted file mode 100644
index 1cff5e055..000000000
--- a/biter/src/blk_metadata_and_block.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-use bitcoin::Block;
-
-use crate::BlkMetadata;
-
-pub struct BlkMetadataAndBlock {
- pub blk_metadata: BlkMetadata,
- pub block: Block,
-}
-
-impl BlkMetadataAndBlock {
- pub fn new(blk_metadata: BlkMetadata, block: Block) -> Self {
- Self {
- blk_metadata,
- block,
- }
- }
-}
diff --git a/biter/src/blk_recap.rs b/biter/src/blk_recap.rs
deleted file mode 100644
index 26e35daa9..000000000
--- a/biter/src/blk_recap.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use std::path::PathBuf;
-
-use bitcoin::{hashes::Hash, BlockHash};
-use serde::{Deserialize, Serialize};
-
-use crate::{path_to_modified_time, BlkMetadataAndBlock};
-
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
-pub struct BlkRecap {
- min_continuous_height: usize,
- min_continuous_prev_hash: BlockHash,
- modified_time: u64,
-}
-
-impl BlkRecap {
- pub fn first(blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
- Self {
- min_continuous_height: 0,
- min_continuous_prev_hash: BlockHash::all_zeros(),
- modified_time: blk_metadata_and_block.blk_metadata.modified_time,
- }
- }
-
- pub fn from(height: usize, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
- Self {
- min_continuous_height: height,
- min_continuous_prev_hash: blk_metadata_and_block.block.header.prev_blockhash,
- modified_time: blk_metadata_and_block.blk_metadata.modified_time,
- }
- }
-
- pub fn has_different_modified_time(&self, blk_path: &PathBuf) -> bool {
- self.modified_time != path_to_modified_time(blk_path)
- }
-
- pub fn is_younger_than(&self, height: usize) -> bool {
- self.min_continuous_height > height
- }
-
- pub fn height(&self) -> usize {
- self.min_continuous_height
- }
-
- pub fn prev_hash(&self) -> &BlockHash {
- &self.min_continuous_prev_hash
- }
-}
diff --git a/biter/src/lib.rs b/biter/src/lib.rs
deleted file mode 100644
index a064fa1cb..000000000
--- a/biter/src/lib.rs
+++ /dev/null
@@ -1,384 +0,0 @@
-use std::{
- collections::{BTreeMap, BTreeSet, VecDeque},
- fs::{self},
- ops::ControlFlow,
- thread,
-};
-
-use bitcoin::{
- consensus::{Decodable, ReadExt},
- hashes::Hash,
- io::{Cursor, Read},
- Block, BlockHash,
-};
-use bitcoincore_rpc::RpcApi;
-use crossbeam::channel::{bounded, Receiver};
-use rayon::prelude::*;
-
-pub use bitcoin;
-pub use bitcoincore_rpc;
-
-mod blk_index_to_blk_recap;
-mod blk_metadata;
-mod blk_metadata_and_block;
-mod blk_recap;
-mod utils;
-
-use blk_index_to_blk_recap::*;
-use blk_metadata::*;
-use blk_metadata_and_block::*;
-use utils::*;
-
-pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
-const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
-const BOUND_CAP: usize = 210;
-
-enum BlockState {
- Raw(Vec),
- Decoded(Block),
-}
-
-///
-/// Returns a crossbeam channel receiver that receives `(usize, Block, BlockHash)` tuples (with `usize` being the height) in sequential order.
-///
-/// # Arguments
-///
-/// * `data_dir` - Path to the Bitcoin data directory
-/// * `export_dir` - Path to the export directory where a mini blk indexer will be exported
-/// * `start` - Inclusive starting height of the blocks received, `None` for 0
-/// * `end` - Inclusive ending height of the blocks received, `None` for the last one
-/// * `rpc` - RPC client to filter out forks
-///
-/// # Example
-///
-/// ```rust
-/// use bitcoincore_rpc::{Auth, Client};
-///
-/// fn main() {
-/// let i = std::time::Instant::now();
-///
-/// let url = "http://localhost:8332";
-/// let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
-/// let rpc = Client::new(url, auth).unwrap();
-///
-/// let data_dir = "../../bitcoin";
-/// let export_dir = "./target";
-/// let start = Some(850_000);
-/// let end = None;
-///
-/// biter::new(data_dir, export_dir, start, end, rpc)
-/// .iter()
-/// .for_each(|(height, _block, hash)| {
-/// println!("{height}: {hash}");
-/// });
-///
-/// dbg!(i.elapsed());
-///}
-/// ```
-///
-pub fn new(
- data_dir: &str,
- export_dir: &str,
- start: Option,
- end: Option,
- rpc: bitcoincore_rpc::Client,
-) -> Receiver<(usize, Block, BlockHash)> {
- let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP);
- let (send_block, recv_block) = bounded(BOUND_CAP);
- let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
-
- let blocks_dir = scan_blocks_dir(data_dir);
-
- let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blocks_dir, export_dir);
-
- let start_recap = blk_index_to_blk_recap.get_start_recap(start);
- let starting_blk_index = start_recap.as_ref().map_or(0, |(index, _)| *index);
-
- thread::spawn(move || {
- blocks_dir
- .into_iter()
- .filter(|(blk_index, _)| blk_index >= &starting_blk_index)
- .try_for_each(move |(blk_index, blk_path)| {
- let blk_metadata = BlkMetadata::new(blk_index, &blk_path);
-
- let blk_bytes = fs::read(&blk_path).unwrap();
- let blk_bytes_len = blk_bytes.len() as u64;
-
- let mut cursor = Cursor::new(blk_bytes.as_slice());
-
- let mut current_4bytes = [0; 4];
-
- 'parent: loop {
- if cursor.position() == blk_bytes_len {
- break;
- }
-
- // Read until we find a valid suite of MAGIC_BYTES
- loop {
- current_4bytes.rotate_left(1);
-
- if let Ok(byte) = cursor.read_u8() {
- current_4bytes[3] = byte;
- } else {
- break 'parent;
- }
-
- if current_4bytes == MAGIC_BYTES {
- break;
- }
- }
-
- let block_size = cursor.read_u32().unwrap();
-
- let mut raw_block = vec![0u8; block_size as usize];
-
- cursor.read_exact(&mut raw_block).unwrap();
-
- if send_block_reader
- .send((blk_metadata, BlockState::Raw(raw_block)))
- .is_err()
- {
- return ControlFlow::Break(());
- }
- }
-
- ControlFlow::Continue(())
- })
- });
-
- // thread::spawn(move || {
- // recv_block_reader.iter().par_bridge().try_for_each(
- // move |(blk_metadata, mut block_state)| {
- // let raw_block = match block_state {
- // BlockState::Raw(vec) => vec,
- // _ => unreachable!(),
- // };
-
- // let mut cursor = Cursor::new(raw_block);
-
- // block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
-
- // if send_block
- // .send(BlkMetadataAndBlock::new(
- // blk_metadata,
- // match block_state {
- // BlockState::Decoded(block) => block,
- // _ => unreachable!(),
- // },
- // ))
- // .is_err()
- // {
- // return ControlFlow::Break(());
- // }
-
- // ControlFlow::Continue(())
- // },
- // );
- // });
-
- // Can't use the previous code because .send() blocks all the threads if full
- // And other .par_iter() are also stuck because of that
- thread::spawn(move || {
- let mut bulk = vec![];
-
- let drain_and_send = |bulk: &mut Vec<_>| {
- // Using a vec and sending after to not end up with stuck threads in par iter
- bulk.par_iter_mut().for_each(|(_, block_state)| {
- let raw_block = match block_state {
- BlockState::Raw(vec) => vec,
- _ => unreachable!(),
- };
-
- let mut cursor = Cursor::new(raw_block);
-
- *block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
- });
-
- bulk.drain(..).try_for_each(|(blk_metadata, block_state)| {
- let block = match block_state {
- BlockState::Decoded(block) => block,
- _ => unreachable!(),
- };
-
- if send_block
- .send(BlkMetadataAndBlock::new(blk_metadata, block))
- .is_err()
- {
- return ControlFlow::Break(());
- }
-
- ControlFlow::Continue(())
- })
- };
-
- recv_block_reader.iter().try_for_each(|tuple| {
- bulk.push(tuple);
-
- if bulk.len() < BOUND_CAP / 2 {
- return ControlFlow::Continue(());
- }
-
- drain_and_send(&mut bulk)
- });
-
- drain_and_send(&mut bulk)
- });
-
- // Tokio version: 1022s
- // Slighlty slower than rayon version
- // thread::spawn(move || {
- // let rt = tokio::runtime::Runtime::new().unwrap();
- // let _guard = rt.enter();
-
- // let mut tasks = VecDeque::with_capacity(BOUND);
-
- // recv_block_reader
- // .iter()
- // .try_for_each(move |(blk_metadata, block_state)| {
- // let raw_block = match block_state {
- // BlockState::Raw(vec) => vec,
- // _ => unreachable!(),
- // };
-
- // tasks.push_back(tokio::task::spawn(async move {
- // let block = Block::consensus_decode(&mut Cursor::new(raw_block)).unwrap();
-
- // (blk_metadata, block)
- // }));
-
- // while tasks.len() > BOUND {
- // let (blk_metadata, block) = rt.block_on(tasks.pop_front().unwrap()).unwrap();
-
- // if send_block
- // .send(BlkMetadataAndBlock::new(blk_metadata, block))
- // .is_err()
- // {
- // return ControlFlow::Break(());
- // }
- // }
-
- // ControlFlow::Continue(())
- // });
- //
- // todo!("Send the rest")
- // });
-
- thread::spawn(move || {
- let mut height = start_recap.map_or(0, |(_, recap)| recap.height());
-
- let mut future_blocks = BTreeMap::default();
- let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default();
- let mut recent_hashes: BTreeSet = BTreeSet::default();
-
- let mut prev_hash =
- start_recap.map_or_else(BlockHash::all_zeros, |(_, recap)| *recap.prev_hash());
-
- let mut prepare_and_send = |(hash, tuple): (BlockHash, BlkMetadataAndBlock)| {
- blk_index_to_blk_recap.update(&tuple, height);
-
- if start.map_or(true, |start| start <= height) {
- send_height_block_hash
- .send((height, tuple.block, hash))
- .unwrap();
- }
-
- if end.map_or(false, |end| height == end) {
- return ControlFlow::Break(());
- }
-
- height += 1;
-
- ControlFlow::Continue(())
- };
-
- let mut update_tip = |prev_hash: &mut BlockHash,
- recent_hashes: &mut BTreeSet,
- recent_chain: &mut VecDeque<(BlockHash, BlkMetadataAndBlock)>,
- future_blocks: &mut BTreeMap,
- tuple: BlkMetadataAndBlock| {
- let mut tuple = Some(tuple);
-
- while let Some(tuple) = tuple.take().or_else(|| future_blocks.remove(prev_hash)) {
- let hash = tuple.block.block_hash();
-
- *prev_hash = hash;
- recent_hashes.insert(hash);
- recent_chain.push_back((hash, tuple));
- }
-
- while recent_chain.len() > NUMBER_OF_UNSAFE_BLOCKS {
- let (hash, tuple) = recent_chain.pop_front().unwrap();
-
- recent_hashes.remove(&hash);
-
- if prepare_and_send((hash, tuple)).is_break() {
- return ControlFlow::Break(());
- }
- }
-
- ControlFlow::Continue(())
- };
-
- let flow = recv_block.iter().try_for_each(|tuple| {
- // block isn't next after current tip
- if prev_hash != tuple.block.header.prev_blockhash {
- let is_block_active =
- |hash| rpc.get_block_header_info(hash).unwrap().confirmations > 0;
-
- // block prev has already been processed
- if recent_hashes.contains(&tuple.block.header.prev_blockhash) {
- let hash = tuple.block.block_hash();
-
- if is_block_active(&hash) {
- let prev_index = recent_chain
- .iter()
- .position(|(hash, ..)| hash == &tuple.block.header.prev_blockhash)
- .unwrap();
-
- let bad_index_start = prev_index + 1;
-
- recent_chain.drain(bad_index_start..).for_each(|(hash, _)| {
- recent_hashes.remove(&hash);
- });
-
- return update_tip(
- &mut prev_hash,
- &mut recent_hashes,
- &mut recent_chain,
- &mut future_blocks,
- tuple,
- );
- }
- // Check if there was already a future block with the same prev hash
- } else if let Some(prev_tuple) =
- future_blocks.insert(tuple.block.header.prev_blockhash, tuple)
- {
- // If the previous was the active one
- if is_block_active(&prev_tuple.block.block_hash()) {
- // Rollback the insert
- future_blocks.insert(prev_tuple.block.header.prev_blockhash, prev_tuple);
- }
- }
- } else {
- return update_tip(
- &mut prev_hash,
- &mut recent_hashes,
- &mut recent_chain,
- &mut future_blocks,
- tuple,
- );
- }
-
- ControlFlow::Continue(())
- });
-
- if flow.is_continue() {
- // Send the last (up to 100) blocks
- recent_chain.into_iter().try_for_each(prepare_and_send);
- }
-
- blk_index_to_blk_recap.export();
- });
-
- recv_height_block_hash
-}
diff --git a/biter/src/main.rs b/biter/src/main.rs
deleted file mode 100644
index 57ac1f020..000000000
--- a/biter/src/main.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-use bitcoincore_rpc::{Auth, Client};
-
-fn main() {
- let i = std::time::Instant::now();
-
- let url = "http://localhost:8332";
- let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
- let rpc = Client::new(url, auth).unwrap();
-
- let data_dir = "../bitcoin";
- let export_dir = "./target";
- let start = None;
- let end = None;
-
- biter::new(data_dir, export_dir, start, end, rpc)
- .iter()
- .for_each(|(height, _block, hash)| {
- println!("{height}: {hash}");
- });
-
- dbg!(i.elapsed());
-}
diff --git a/biter/src/utils.rs b/biter/src/utils.rs
deleted file mode 100644
index 9b1bdb945..000000000
--- a/biter/src/utils.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-use std::{collections::BTreeMap, fs, path::PathBuf, time::UNIX_EPOCH};
-
-const BLK: &str = "blk";
-const DAT: &str = ".dat";
-
-pub fn scan_blocks_dir(data_dir_path: &str) -> BTreeMap {
- let blocks_dir_path = &format!("{data_dir_path}/blocks");
-
- fs::read_dir(blocks_dir_path)
- .unwrap()
- .map(|entry| entry.unwrap().path())
- .filter(|path| {
- let is_file = path.is_file();
-
- if is_file {
- let file_name = path.file_name().unwrap().to_str().unwrap();
-
- file_name.starts_with(BLK) && file_name.ends_with(DAT)
- } else {
- false
- }
- })
- .map(|path| {
- let file_name = path.file_name().unwrap().to_str().unwrap();
-
- let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())]
- .parse::()
- .unwrap();
-
- (blk_index, path)
- })
- .collect::>()
-}
-
-pub fn path_to_modified_time(path: &PathBuf) -> u64 {
- fs::metadata(path)
- .unwrap()
- .modified()
- .unwrap()
- .duration_since(UNIX_EPOCH)
- .unwrap()
- .as_secs()
-}
diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml
new file mode 100644
index 000000000..9dd70b5dc
--- /dev/null
+++ b/crates/brk/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "brk"
+description = "The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your node"
+license.workspace = true
+edition.workspace = true
+version.workspace = true
+
+[features]
+full = ["computer", "fetcher", "indexer", "logger", "parser", "server"]
+computer = ["brk_computer"]
+fetcher = ["brk_fetcher"]
+indexer = ["brk_indexer"]
+logger = ["brk_logger"]
+parser = ["brk_parser"]
+server = ["brk_server"]
+
+[dependencies]
+brk_computer = { workspace = true, optional = true }
+brk_fetcher = { workspace = true, optional = true }
+brk_indexer = { workspace = true, optional = true }
+brk_logger = { workspace = true, optional = true }
+brk_parser = { workspace = true, optional = true }
+brk_server = { workspace = true, optional = true }
+
+[package.metadata.docs.rs]
+all-features = true
diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs
new file mode 100644
index 000000000..458340dca
--- /dev/null
+++ b/crates/brk/src/lib.rs
@@ -0,0 +1,35 @@
+#[cfg(feature = "computer")]
+pub mod computer {
+ #[doc(inline)]
+ pub use brk_computer::*;
+}
+
+#[cfg(feature = "fetcher")]
+pub mod fetcher {
+ #[doc(inline)]
+ pub use brk_fetcher::*;
+}
+
+#[cfg(feature = "indexer")]
+pub mod indexer {
+ #[doc(inline)]
+ pub use brk_indexer::*;
+}
+
+#[cfg(feature = "logger")]
+pub mod logger {
+ #[doc(inline)]
+ pub use brk_logger::*;
+}
+
+#[cfg(feature = "parser")]
+pub mod parser {
+ #[doc(inline)]
+ pub use brk_parser::*;
+}
+
+#[cfg(feature = "server")]
+pub mod server {
+ #[doc(inline)]
+ pub use brk_server::*;
+}
diff --git a/crates/brk/src/main.rs b/crates/brk/src/main.rs
new file mode 100644
index 000000000..e7a11a969
--- /dev/null
+++ b/crates/brk/src/main.rs
@@ -0,0 +1,3 @@
+fn main() {
+ println!("Hello, world!");
+}
diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml
new file mode 100644
index 000000000..f4ca6df65
--- /dev/null
+++ b/crates/brk_cli/Cargo.toml
@@ -0,0 +1,8 @@
+[package]
+name = "brk_cli"
+description = "A command line interface to interact with the Bitcoin Research Kit"
+version = { workspace = true }
+edition = { workspace = true }
+license = { workspace = true }
+
+[dependencies]
diff --git a/crates/brk_cli/src/main.rs b/crates/brk_cli/src/main.rs
new file mode 100644
index 000000000..e7a11a969
--- /dev/null
+++ b/crates/brk_cli/src/main.rs
@@ -0,0 +1,3 @@
+fn main() {
+ println!("Hello, world!");
+}
diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml
new file mode 100644
index 000000000..add4956ff
--- /dev/null
+++ b/crates/brk_computer/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+name = "brk_computer"
+description = "A Bitcoin dataset computer built on top of brk_indexer and brk_fetcher"
+version = { workspace = true }
+edition = { workspace = true }
+license = { workspace = true }
+
+[dependencies]
+brk_fetcher = { workspace = true }
+brk_indexer = { workspace = true }
+brk_parser = { workspace = true }
+byteview = { workspace = true }
+color-eyre = { workspace = true }
+derive_deref = { workspace = true }
+fjall = { workspace = true }
+hodor = { workspace = true }
+storable_vec = { workspace = true }
+zerocopy = { workspace = true }
diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs
new file mode 100644
index 000000000..5ae619db4
--- /dev/null
+++ b/crates/brk_computer/src/lib.rs
@@ -0,0 +1,114 @@
+use std::path::{Path, PathBuf};
+
+use brk_indexer::Indexer;
+pub use brk_parser::rpc;
+use hodor::Exit;
+
+mod storage;
+mod structs;
+
+use brk_fetcher::Date;
+use storable_vec::SINGLE_THREAD;
+use storage::{Fjalls, StorableVecs};
+pub use structs::*;
+
+pub struct Computer