mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-08 06:01:57 -07:00
global: snapshot
This commit is contained in:
nym21
+3
-25
@@ -1,25 +1,9 @@
|
||||
# Mac OS
|
||||
.DS_Store
|
||||
|
||||
# To do
|
||||
/charts
|
||||
TODO.md
|
||||
|
||||
# Builds
|
||||
dist
|
||||
target
|
||||
|
||||
# I/O
|
||||
in
|
||||
out
|
||||
.log
|
||||
/datasets
|
||||
/datasets2
|
||||
/price
|
||||
*..*
|
||||
/txout_*
|
||||
/db
|
||||
|
||||
# Sync
|
||||
.stfolder
|
||||
|
||||
@@ -29,16 +13,10 @@ out
|
||||
# Ignored
|
||||
ignore
|
||||
|
||||
# Scripts
|
||||
/start-node.sh
|
||||
|
||||
# Editors
|
||||
.vscode
|
||||
.zed
|
||||
|
||||
# Configs
|
||||
config.toml
|
||||
|
||||
# Flamegraph
|
||||
flamegraph/
|
||||
flamegraph.svg
|
||||
@@ -53,7 +31,7 @@ snapshots*/
|
||||
docker/kibo
|
||||
|
||||
# Types
|
||||
website/scripts/types/paths.d.ts
|
||||
paths.d.ts
|
||||
|
||||
# Misc
|
||||
OPENSATS.md
|
||||
# Git
|
||||
.git*
|
||||
|
||||
+38
-31
@@ -1,13 +1,20 @@
|
||||
# Changelog
|
||||
|
||||
<!--
|
||||
## v. 0.X.Y | WIP
|
||||
|
||||
# v0.X.Y | WIP
|
||||
|
||||
-->
|
||||
|
||||
## v. 0.5.0 | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
# v0.6.0 | WIP
|
||||
|
||||
|
||||
- Merged parser and server crates into a single project (and thus executable)
|
||||
- Started using `log` and `env_logger` crates
|
||||
- Improved logs
|
||||
- Added `--server BOOL` and `--parser BOOL` parameters (both are true by default)
|
||||
- Automated databases defragmention (and removed parameter)
|
||||
- Fixed input being unfocused right after being focused in Brave browser
|
||||
|
||||
# [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
|
||||
|
||||
|
||||
## Datasets
|
||||
|
||||
@@ -72,15 +79,15 @@
|
||||
|
||||
- Moved back to this repo
|
||||
|
||||
## v. 0.4.0 | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
# [v0.4.0](https://github.com/kibo-money/kibo/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
|
||||
|
||||
|
||||
|
||||
### Brand
|
||||
## Brand
|
||||
|
||||
- **Satonomics** is now **kibō** 🎉
|
||||
|
||||
### Website
|
||||
## Website
|
||||
|
||||
- Complete redesign of the website
|
||||
- Rewrote the whole application and removed `node`/`npm`/`pnpm` dependencies in favor for pure `HTML`/`CSS`/`Javascript`
|
||||
@@ -88,7 +95,7 @@
|
||||
- Added Trading View attribution link to the settings frame and file in the lightweight charts folder
|
||||
- Many other changes
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
|
||||
- Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage
|
||||
@@ -103,17 +110,17 @@
|
||||
- Various first run fixes
|
||||
- Added to `-h` which arguments are saved, which is all of them at the time of writing
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Updated the code to support compressed binaries
|
||||
- Added serving of the website
|
||||
- Improved `Cache-Control` behavior
|
||||
|
||||
## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
|
||||
# [v0.3.0](https://github.com/kibo-money/kibo/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
|
||||
|
||||
|
||||
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Global
|
||||
- Improved self-hosting by:
|
||||
@@ -156,7 +163,7 @@
|
||||
- Price
|
||||
- Improved error message when price cannot be found
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added chart scroll button for nice animations à la Wicked
|
||||
@@ -182,17 +189,17 @@
|
||||
- Settings
|
||||
- Removed the horizontal scroll bar which was unintended
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Run file
|
||||
- Only run with a watcher if `cargo watch` is available
|
||||
- Removed id_to_path file in favor for only `paths.d.ts` in `app/src/types`
|
||||
|
||||
## v. 0.2.0 | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
|
||||
# [v0.2.0](https://github.com/kibo-money/kibo/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
|
||||
|
||||
|
||||
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added the height version of all datasets and many optimizations to make them usable but only available on desktop and tablets for now
|
||||
@@ -220,24 +227,24 @@
|
||||
- Hopefully made scrollbars a little more subtle on WIndows and Linux, can't test
|
||||
- Generale style updates
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Fixed ulimit only being run in Mac OS instead of whenever the program is detected
|
||||
|
||||
## v. 0.1.1 | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
|
||||
# [v0.1.1](https://github.com/kibo-money/kibo/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
|
||||
|
||||
|
||||
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Fixed overflow in `Price` struct which caused many Realized Caps and Realized Prices to have completely bogus data
|
||||
- Fixed Realized Cap computation which was using rounded prices instead normal ones
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Added the chunk, date and time of the request to the terminal logs
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- Chart
|
||||
- Added double click option on a legend to toggle the visibility of all other series
|
||||
@@ -270,14 +277,14 @@
|
||||
- Misc
|
||||
- Removed tracker even though it was a very privacy friendly as it appeared to not be working properly
|
||||
|
||||
### Price
|
||||
## Price
|
||||
|
||||
- Deleted old price datasets and their backups
|
||||
|
||||
## v. 0.1.0 | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
|
||||
# [v0.1.0](https://github.com/kibo-money/kibo/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
|
||||
|
||||
|
||||
|
||||
|
||||
## v. 0.0.X | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
|
||||
# v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
|
||||
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -3,6 +3,6 @@
|
||||
## Parser
|
||||
|
||||
- Avoid floats as much as possible
|
||||
- Use structs like `WAmount` and `Price` for calculations
|
||||
- **Only** use `WAmount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
|
||||
- Use structs like `Amount` and `Price` for calculations
|
||||
- **Only** use `Amount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
|
||||
- No `Arc`, `Rc`, `Mutex` even from third party libraries, they're slower
|
||||
|
||||
Generated
+290
-266
File diff suppressed because it is too large
Load Diff
+39
@@ -0,0 +1,39 @@
|
||||
[package]
|
||||
name = "kibo_money"
|
||||
version = "0.6.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
allocative = "0.3.3"
|
||||
axum = "0.7.9"
|
||||
bincode = { git = "https://github.com/bincode-org/bincode.git", features = [
|
||||
"serde",
|
||||
] }
|
||||
bitcoin_hashes = { version = "0.15.0" }
|
||||
biter = { path = "./crates/biter" }
|
||||
chrono = { version = "0.4.39", features = ["serde"] }
|
||||
clap = { version = "4.5.23", features = ["derive"] }
|
||||
color-eyre = "0.6.3"
|
||||
ctrlc = { version = "3.4.5", features = ["termination"] }
|
||||
derive_deref = "1.1.1"
|
||||
env_logger = "0.11.5"
|
||||
inferno = "0.12.0"
|
||||
itertools = "0.13.0"
|
||||
log = { version = "0.4.22", features = ["std", "serde"] }
|
||||
ordered-float = "4.5.0"
|
||||
rayon = "1.10.0"
|
||||
regex = "1.11.1"
|
||||
reqwest = { version = "0.12.9", features = ["blocking", "json"] }
|
||||
rlimit = "0.10.2"
|
||||
sanakirja = "1.4.3"
|
||||
serde = { version = "1.0.216", features = ["derive"] }
|
||||
serde_json = "1.0.133"
|
||||
struct_iterable = { path = "./crates/iterable" }
|
||||
swc = "9.0.0"
|
||||
swc_common = "5.0.0"
|
||||
tokio = { version = "1.42.0", features = ["full"] }
|
||||
toml = "0.8.19"
|
||||
tower-http = { version = "0.6.2", features = ["compression-full"] }
|
||||
zstd = "0.13.2"
|
||||
@@ -122,6 +122,7 @@ Now we can finally start by running the parser, you need to use the `./run.sh` s
|
||||
For the first launch, the parser will need several information such as:
|
||||
|
||||
- `--datadir`: which is bitcoin data directory path, prefer `$HOME` to `~` as the latter might not work
|
||||
- `--outdir`: where all outputs will be saved, prefer `$HOME` to `~` as the latter might not work
|
||||
|
||||
Optionally you can also specify:
|
||||
|
||||
@@ -136,7 +137,7 @@ Everything will be saved in a `config.toml` file, which will allow you to simply
|
||||
Here's an example
|
||||
|
||||
```bash
|
||||
./run.sh --datadir=$HOME/Developer/bitcoin
|
||||
./run.sh --datadir=$HOME/Developer/bitcoin --outdir=$HOME/.kibo/out
|
||||
```
|
||||
|
||||
In a **new** terminal, go to the `server`'s folder of the repository
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# v0.2.1
|
||||
|
||||
- Clean `.json` if necessary
|
||||
- Only save `.json` if needed
|
||||
- Updated benchmarks
|
||||
- Updated packages
|
||||
|
||||
# v0.2.0
|
||||
|
||||
- Removed the need for an output directory path
|
||||
- Changed the location of the saved json file from the previously needed output directory path to the Bitcoin data directory
|
||||
- Added a save of the json file every 144 * 30 blocks instead of only at the end
|
||||
+44
-35
@@ -1,12 +1,12 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.4"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
|
||||
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
|
||||
|
||||
[[package]]
|
||||
name = "base58ck"
|
||||
@@ -32,9 +32,9 @@ checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin"
|
||||
version = "0.32.2"
|
||||
version = "0.32.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea507acc1cd80fc084ace38544bbcf7ced7c2aa65b653b102de0ce718df668f6"
|
||||
checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026"
|
||||
dependencies = [
|
||||
"base58ck",
|
||||
"bech32",
|
||||
@@ -110,7 +110,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "biter"
|
||||
version = "0.1.1"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"bitcoincore-rpc",
|
||||
@@ -129,9 +129,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.1.6"
|
||||
version = "1.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f"
|
||||
checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f"
|
||||
dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
@@ -258,9 +261,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.155"
|
||||
version = "0.2.161"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
|
||||
checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
@@ -287,27 +290,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.18"
|
||||
version = "0.2.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f"
|
||||
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.86"
|
||||
version = "1.0.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
|
||||
checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.36"
|
||||
version = "1.0.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
|
||||
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -370,9 +373,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
||||
|
||||
[[package]]
|
||||
name = "secp256k1"
|
||||
version = "0.29.0"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e0cc0f1cf93f4969faf3ea1c7d8a9faed25918d96affa959720823dfe86d4f3"
|
||||
checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113"
|
||||
dependencies = [
|
||||
"bitcoin_hashes",
|
||||
"rand",
|
||||
@@ -382,27 +385,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "secp256k1-sys"
|
||||
version = "0.10.0"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1433bd67156263443f14d603720b082dd3121779323fce20cba2aa07b874bc1b"
|
||||
checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.204"
|
||||
version = "1.0.216"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
|
||||
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.204"
|
||||
version = "1.0.216"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
|
||||
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -411,9 +414,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.122"
|
||||
version = "1.0.133"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
|
||||
checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
@@ -422,10 +425,16 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.72"
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.90"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -434,9 +443,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
@@ -446,9 +455,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.6.6"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6"
|
||||
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"zerocopy-derive",
|
||||
@@ -456,9 +465,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.6.6"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
|
||||
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1,19 +1,19 @@
|
||||
[package]
|
||||
name = "biter"
|
||||
description = "A very fast Bitcoin block iterator"
|
||||
version = "0.1.1"
|
||||
version = "0.2.1"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/kibo-money/kibo/tree/main/biter"
|
||||
repository = "https://github.com/kibo-money/kibo/tree/main/crates/biter"
|
||||
keywords = ["bitcoin", "block", "iterator"]
|
||||
categories = ["cryptography::cryptocurrencies", "encoding"]
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
bitcoin = { version = "0.32.2", features = ["serde"] }
|
||||
bitcoin = { version = "0.32.5", features = ["serde"] }
|
||||
rayon = "1.10.0"
|
||||
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
|
||||
serde = { version = "1.0.204", features = ["derive"] }
|
||||
serde_json = "1.0.122"
|
||||
serde = { version = "1.0.216", features = ["derive"] }
|
||||
serde_json = "1.0.133"
|
||||
derived-deref = "2.1.0"
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
# tokio = { version = "1.39.2", features = ["rt-multi-thread"] }
|
||||
@@ -10,6 +10,8 @@ The element returned by the iterator is a tuple which includes the:
|
||||
## Example
|
||||
|
||||
```rust
|
||||
use std::path::Path;
|
||||
|
||||
use bitcoincore_rpc::{Auth, Client};
|
||||
|
||||
fn main() {
|
||||
@@ -18,9 +20,6 @@ fn main() {
|
||||
// Path to the Bitcoin data directory
|
||||
let data_dir = "../../bitcoin";
|
||||
|
||||
// Path to the export directory where a mini blk indexer will be exported
|
||||
let export_dir = "./target";
|
||||
|
||||
// Inclusive starting height of the blocks received, `None` for 0
|
||||
let start = Some(850_000);
|
||||
|
||||
@@ -29,11 +28,15 @@ fn main() {
|
||||
|
||||
// RPC client to filter out forks
|
||||
let url = "http://localhost:8332";
|
||||
let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
let cookie = Path::new(data_dir).join(".cookie");
|
||||
let auth = Auth::CookieFile(cookie);
|
||||
let rpc = Client::new(url, auth).unwrap();
|
||||
|
||||
if cookie.is_file() {
|
||||
Ok()
|
||||
|
||||
// Create channel receiver then iterate over the blocks
|
||||
biter::new(data_dir, export_dir, start, end, rpc)
|
||||
biter::new(data_dir, start, end, rpc)
|
||||
.iter()
|
||||
.for_each(|(height, _block, hash)| {
|
||||
println!("{height}: {hash}");
|
||||
@@ -52,11 +55,11 @@ Peak memory should be around 500MB.
|
||||
|
||||
## Comparaison
|
||||
|
||||
| | [biter](https://crates.io/crates/biter) | [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) | [blocks_iterator](https://crates.io/crates/blocks_iterator) |
|
||||
| | [biter](https://crates.io/crates/biter) | [bitcoin-explorer (depreciated)](https://crates.io/crates/bitcoin-explorer) | [blocks_iterator](https://crates.io/crates/blocks_iterator) |
|
||||
| --- | --- | --- | --- |
|
||||
| Run **with** `bitcoind` | Yes ✅ | No ❌ | Yes ✅ |
|
||||
| Run **without** `bitcoind` | No ❌ | Yes ✅ | Yes ✅ |
|
||||
| `0..=855_000` | 16mn40s | 17mn 46s | > 2h |
|
||||
| `800_000..=855_000` | 2mn 53s (16mn40s if first run) | 3mn 2s | > 2h |
|
||||
| Runs **with** `bitcoind` | Yes ✅ | No ❌ | Yes ✅ |
|
||||
| Runs **without** `bitcoind` | No ❌ | Yes ✅ | Yes ✅ |
|
||||
| `0..=855_000` | 4mn 10s | 4mn 45s | > 2h |
|
||||
| `800_000..=855_000` | 0mn 52s (4mn 10s if first run) | 0mn 55s | > 2h |
|
||||
|
||||
*Benchmarked on a Macbook Pro M3 Pro*
|
||||
@@ -1,28 +1,31 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::BTreeMap,
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fs::{self, File},
|
||||
io::{BufReader, BufWriter},
|
||||
path::PathBuf,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use derived_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{blk_recap::BlkRecap, BlkMetadataAndBlock};
|
||||
|
||||
const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30;
|
||||
|
||||
#[derive(Deref, DerefMut, Debug)]
|
||||
pub struct BlkIndexToBlkRecap {
|
||||
path: String,
|
||||
path: PathBuf,
|
||||
#[target]
|
||||
tree: BTreeMap<usize, BlkRecap>,
|
||||
last_safe_recap: Option<BlkRecap>,
|
||||
}
|
||||
|
||||
impl BlkIndexToBlkRecap {
|
||||
pub fn import(blocks_dir: &BTreeMap<usize, PathBuf>, export_dir: &str) -> Self {
|
||||
let path = format!("{export_dir}/blk_index_to_blk_recap.json");
|
||||
pub fn import(blocks_dir: &BTreeMap<usize, PathBuf>, data_dir: &Path) -> Self {
|
||||
let path = data_dir.join("blk_index_to_blk_recap.json");
|
||||
|
||||
let tree = {
|
||||
fs::create_dir_all(export_dir).unwrap();
|
||||
fs::create_dir_all(data_dir).unwrap();
|
||||
|
||||
if let Ok(file) = File::open(&path) {
|
||||
let reader = BufReader::new(file);
|
||||
@@ -32,7 +35,11 @@ impl BlkIndexToBlkRecap {
|
||||
}
|
||||
};
|
||||
|
||||
let mut this = Self { path, tree };
|
||||
let mut this = Self {
|
||||
path,
|
||||
tree,
|
||||
last_safe_recap: None,
|
||||
};
|
||||
|
||||
this.clean_outdated(blocks_dir);
|
||||
|
||||
@@ -40,13 +47,22 @@ impl BlkIndexToBlkRecap {
|
||||
}
|
||||
|
||||
pub fn clean_outdated(&mut self, blocks_dir: &BTreeMap<usize, PathBuf>) {
|
||||
let mut unprocessed_keys = self.keys().copied().collect::<BTreeSet<_>>();
|
||||
|
||||
blocks_dir.iter().for_each(|(blk_index, blk_path)| {
|
||||
unprocessed_keys.remove(blk_index);
|
||||
if let Some(blk_recap) = self.get(blk_index) {
|
||||
if blk_recap.has_different_modified_time(blk_path) {
|
||||
self.remove(blk_index);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
unprocessed_keys.iter().for_each(|blk_index| {
|
||||
self.remove(blk_index);
|
||||
});
|
||||
|
||||
self.last_safe_recap = self.last_entry().map(|e| e.get().clone());
|
||||
}
|
||||
|
||||
pub fn get_start_recap(&self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
|
||||
@@ -73,7 +89,6 @@ impl BlkIndexToBlkRecap {
|
||||
let blk_index = blk_metadata_and_block.blk_metadata.index;
|
||||
|
||||
if let Some(last_entry) = self.last_entry() {
|
||||
// if last_entry.get().is_older_than(height) {
|
||||
match last_entry.key().cmp(&blk_index) {
|
||||
Ordering::Greater => {
|
||||
last_entry.remove_entry();
|
||||
@@ -83,7 +98,6 @@ impl BlkIndexToBlkRecap {
|
||||
}
|
||||
Ordering::Equal => {}
|
||||
};
|
||||
// }
|
||||
} else {
|
||||
if blk_index != 0 || height != 0 {
|
||||
// dbg!(blk_index, height);
|
||||
@@ -92,6 +106,14 @@ impl BlkIndexToBlkRecap {
|
||||
|
||||
self.insert(blk_index, BlkRecap::first(blk_metadata_and_block));
|
||||
}
|
||||
|
||||
if self
|
||||
.last_safe_recap
|
||||
.map_or(true, |recap| recap.height() >= height)
|
||||
&& (height % TARGET_BLOCKS_PER_MONTH) == 0
|
||||
{
|
||||
self.export();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn export(&self) {
|
||||
@@ -2,6 +2,7 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet, VecDeque},
|
||||
fs::{self},
|
||||
ops::ControlFlow,
|
||||
path::Path,
|
||||
thread,
|
||||
};
|
||||
|
||||
@@ -44,7 +45,6 @@ enum BlockState {
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `data_dir` - Path to the Bitcoin data directory
|
||||
/// * `export_dir` - Path to the export directory where a mini blk indexer will be exported
|
||||
/// * `start` - Inclusive starting height of the blocks received, `None` for 0
|
||||
/// * `end` - Inclusive ending height of the blocks received, `None` for the last one
|
||||
/// * `rpc` - RPC client to filter out forks
|
||||
@@ -52,21 +52,23 @@ enum BlockState {
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use std::path::Path;
|
||||
///
|
||||
/// use bitcoincore_rpc::{Auth, Client};
|
||||
///
|
||||
/// fn main() {
|
||||
/// let i = std::time::Instant::now();
|
||||
///
|
||||
/// let data_dir = Path::new("../../bitcoin");
|
||||
/// let url = "http://localhost:8332";
|
||||
/// let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
/// let cookie = Path::new(data_dir).join(".cookie");
|
||||
/// let auth = Auth::CookieFile(cookie);
|
||||
/// let rpc = Client::new(url, auth).unwrap();
|
||||
///
|
||||
/// let data_dir = "../../bitcoin";
|
||||
/// let export_dir = "./target";
|
||||
/// let start = Some(850_000);
|
||||
/// let end = None;
|
||||
///
|
||||
/// biter::new(data_dir, export_dir, start, end, rpc)
|
||||
/// biter::new(data_dir, start, end, rpc)
|
||||
/// .iter()
|
||||
/// .for_each(|(height, _block, hash)| {
|
||||
/// println!("{height}: {hash}");
|
||||
@@ -77,8 +79,7 @@ enum BlockState {
|
||||
/// ```
|
||||
///
|
||||
pub fn new(
|
||||
data_dir: &str,
|
||||
export_dir: &str,
|
||||
data_dir: &Path,
|
||||
start: Option<usize>,
|
||||
end: Option<usize>,
|
||||
rpc: bitcoincore_rpc::Client,
|
||||
@@ -87,15 +88,15 @@ pub fn new(
|
||||
let (send_block, recv_block) = bounded(BOUND_CAP);
|
||||
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
|
||||
|
||||
let blocks_dir = scan_blocks_dir(data_dir);
|
||||
let blk_index_to_blk_path = scan_blocks_dir(data_dir);
|
||||
|
||||
let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blocks_dir, export_dir);
|
||||
let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blk_index_to_blk_path, data_dir);
|
||||
|
||||
let start_recap = blk_index_to_blk_recap.get_start_recap(start);
|
||||
let starting_blk_index = start_recap.as_ref().map_or(0, |(index, _)| *index);
|
||||
|
||||
thread::spawn(move || {
|
||||
blocks_dir
|
||||
blk_index_to_blk_path
|
||||
.into_iter()
|
||||
.filter(|(blk_index, _)| blk_index >= &starting_blk_index)
|
||||
.try_for_each(move |(blk_index, blk_path)| {
|
||||
@@ -1,18 +1,20 @@
|
||||
use std::path::Path;
|
||||
|
||||
use bitcoincore_rpc::{Auth, Client};
|
||||
|
||||
fn main() {
|
||||
let i = std::time::Instant::now();
|
||||
|
||||
let data_dir = Path::new("../../../bitcoin");
|
||||
let url = "http://localhost:8332";
|
||||
let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
let cookie = Path::new(data_dir).join(".cookie");
|
||||
let auth = Auth::CookieFile(cookie);
|
||||
let rpc = Client::new(url, auth).unwrap();
|
||||
|
||||
let data_dir = "../bitcoin";
|
||||
let export_dir = "./target";
|
||||
let start = None;
|
||||
let end = None;
|
||||
let start = Some(800_000);
|
||||
let end = Some(855_000);
|
||||
|
||||
biter::new(data_dir, export_dir, start, end, rpc)
|
||||
biter::new(data_dir, start, end, rpc)
|
||||
.iter()
|
||||
.for_each(|(height, _block, hash)| {
|
||||
println!("{height}: {hash}");
|
||||
@@ -1,12 +1,17 @@
|
||||
use std::{collections::BTreeMap, fs, path::PathBuf, time::UNIX_EPOCH};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
time::UNIX_EPOCH,
|
||||
};
|
||||
|
||||
const BLK: &str = "blk";
|
||||
const DAT: &str = ".dat";
|
||||
|
||||
pub fn scan_blocks_dir(data_dir_path: &str) -> BTreeMap<usize, PathBuf> {
|
||||
let blocks_dir_path = &format!("{data_dir_path}/blocks");
|
||||
pub fn scan_blocks_dir(data_dir: &Path) -> BTreeMap<usize, PathBuf> {
|
||||
let blocks_dir = data_dir.join("blocks");
|
||||
|
||||
fs::read_dir(blocks_dir_path)
|
||||
fs::read_dir(blocks_dir)
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.filter(|path| {
|
||||
Generated
-2430
File diff suppressed because it is too large
Load Diff
@@ -1,32 +0,0 @@
|
||||
[package]
|
||||
name = "parser"
|
||||
version = "0.5.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
allocative = "0.3.3"
|
||||
bincode = { git = "https://github.com/bincode-org/bincode.git", features = [
|
||||
"serde",
|
||||
] }
|
||||
bitcoin_hashes = { version = "0.14.0" }
|
||||
biter = { path = "../biter" }
|
||||
chrono = { version = "0.4.38", features = ["serde"] }
|
||||
clap = { version = "4.5.20", features = ["derive"] }
|
||||
color-eyre = "0.6.3"
|
||||
ctrlc = { version = "3.4.5", features = ["termination"] }
|
||||
derive_deref = "1.1.1"
|
||||
inferno = "0.11.21"
|
||||
itertools = "0.13.0"
|
||||
ordered-float = "4.4.0"
|
||||
rayon = "1.10.0"
|
||||
reqwest = { version = "0.12.9", features = ["blocking", "json"] }
|
||||
sanakirja = "1.4.3"
|
||||
serde = { version = "1.0.214", features = ["derive"] }
|
||||
serde_json = "1.0.132"
|
||||
struct_iterable = { path = "../iterable" }
|
||||
toml = "0.8.19"
|
||||
zstd = "0.13.2"
|
||||
# memory-stats = "1.2.0"
|
||||
# sysinfo = "0.32.0"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
echo "Deleting datasets..."
|
||||
rm -r ../datasets
|
||||
echo "Deleting states and databases..."
|
||||
rm -r ./out
|
||||
echo "Done."
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
echo "Increasing limit of opened files..."
|
||||
ulimit -n 1000000
|
||||
|
||||
# Needed because the datasets tree is too big lol
|
||||
echo "Increasing stack size..."
|
||||
ulimit -s $(ulimit -Hs)
|
||||
|
||||
cargo build --profile profiling && samply record ./target/profiling/parser "$HOME/Developer/bitcoin"
|
||||
@@ -1,52 +0,0 @@
|
||||
use std::{fs, io, path::PathBuf};
|
||||
|
||||
use crate::{
|
||||
io::OUTPUTS_FOLDER_PATH,
|
||||
structs::{Date, Height},
|
||||
utils::log,
|
||||
};
|
||||
|
||||
use super::AnyDatabase;
|
||||
|
||||
pub trait AnyDatabaseGroup
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
fn init() -> Self {
|
||||
let s = Self::import();
|
||||
s.create_dir_all().unwrap();
|
||||
s
|
||||
}
|
||||
|
||||
fn import() -> Self;
|
||||
|
||||
fn folder<'a>() -> &'a str;
|
||||
|
||||
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>>;
|
||||
fn open_all(&mut self);
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()>;
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), io::Error>;
|
||||
|
||||
fn remove_dir_all(&self) -> color_eyre::Result<(), io::Error> {
|
||||
fs::remove_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
|
||||
log(&format!("Reset {}", Self::folder()));
|
||||
|
||||
self.reset_metadata();
|
||||
self.remove_dir_all()?;
|
||||
self.create_dir_all()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self);
|
||||
|
||||
fn root() -> PathBuf {
|
||||
let folder = Self::folder();
|
||||
PathBuf::from(format!("{OUTPUTS_FOLDER_PATH}/databases/{folder}"))
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub const INPUTS_FOLDER_PATH: &str = "./in";
|
||||
pub const OUTPUTS_FOLDER_PATH: &str = "./out";
|
||||
@@ -1,19 +0,0 @@
|
||||
mod actions;
|
||||
mod databases;
|
||||
mod datasets;
|
||||
mod io;
|
||||
mod price;
|
||||
mod states;
|
||||
mod structs;
|
||||
mod utils;
|
||||
|
||||
pub use crate::{
|
||||
actions::iter_blocks,
|
||||
databases::{AnyDatabase, Database},
|
||||
io::{Binary, Json, Serialization, COMPRESSED_BIN_EXTENSION, JSON_EXTENSION},
|
||||
structs::{
|
||||
Amount, Config, Date, DateMap, Exit, Height, HeightMap, MapChunkId, MapValue,
|
||||
SerializedBTreeMap, SerializedVec, TxoutIndex, HEIGHT_MAP_CHUNK_SIZE, OHLC,
|
||||
},
|
||||
utils::{create_rpc, log, reset_logs},
|
||||
};
|
||||
@@ -1,36 +0,0 @@
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use biter::bitcoincore_rpc::RpcApi;
|
||||
use parser::{create_rpc, iter_blocks, log, reset_logs, Config, Exit};
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
reset_logs();
|
||||
|
||||
let mut config = Config::import()?;
|
||||
|
||||
let rpc = create_rpc(&config).unwrap();
|
||||
|
||||
let exit = Exit::new();
|
||||
|
||||
loop {
|
||||
let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
|
||||
|
||||
log(&format!("{block_count} blocks found."));
|
||||
|
||||
iter_blocks(&mut config, &rpc, block_count, exit.clone())?;
|
||||
|
||||
if let Some(delay) = config.delay {
|
||||
sleep(Duration::from_secs(delay))
|
||||
}
|
||||
|
||||
log("Waiting for a new block...\n");
|
||||
|
||||
while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
use std::{fmt::Debug, fs, io, path::Path};
|
||||
|
||||
use bincode::{Decode, Encode};
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
|
||||
use crate::{io::OUTPUTS_FOLDER_PATH, Serialization};
|
||||
|
||||
// https://github.com/djkoloski/rust_serialization_benchmark
|
||||
pub trait AnyState
|
||||
where
|
||||
Self: Debug + Encode + Decode + Serialize + DeserializeOwned,
|
||||
{
|
||||
fn name<'a>() -> &'a str;
|
||||
|
||||
fn create_dir_all() -> color_eyre::Result<(), io::Error> {
|
||||
fs::create_dir_all(Self::folder_path())
|
||||
}
|
||||
|
||||
fn folder_path() -> String {
|
||||
format!("{OUTPUTS_FOLDER_PATH}/states")
|
||||
}
|
||||
|
||||
fn full_path() -> String {
|
||||
let name = Self::name();
|
||||
|
||||
let folder_path = Self::folder_path();
|
||||
|
||||
format!("{folder_path}/{name}")
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
|
||||
self.clear();
|
||||
|
||||
fs::remove_file(Self::full_path())
|
||||
}
|
||||
|
||||
fn import() -> color_eyre::Result<Self> {
|
||||
Self::create_dir_all()?;
|
||||
|
||||
Serialization::Binary.import(Path::new(&Self::full_path()))
|
||||
}
|
||||
|
||||
fn export(&self) -> color_eyre::Result<()> {
|
||||
Serialization::Binary.export(Path::new(&Self::full_path()), self)
|
||||
}
|
||||
|
||||
fn clear(&mut self);
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
use std::{
|
||||
fs::{self, OpenOptions},
|
||||
io::Write,
|
||||
};
|
||||
|
||||
use chrono::Local;
|
||||
use color_eyre::owo_colors::OwoColorize;
|
||||
|
||||
const LOG_PATH: &str = "./.log";
|
||||
|
||||
pub fn reset_logs() {
|
||||
let _ = fs::remove_file(LOG_PATH);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn log(str: &str) {
|
||||
let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S -"));
|
||||
|
||||
str.lines()
|
||||
.filter(|line| !line.is_empty())
|
||||
.for_each(|line| {
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(LOG_PATH)
|
||||
.unwrap();
|
||||
|
||||
if let Err(e) = writeln!(file, "{} {}", date_time, line) {
|
||||
eprintln!("Couldn't write to file: {}", e);
|
||||
}
|
||||
|
||||
println!("{} {}", date_time.bright_black(), line);
|
||||
});
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
if command -v ulimit &> /dev/null; then
|
||||
echo "Increasing limit of opened files..."
|
||||
ulimit -n 1000000 # Can't be $(ulimit -Hn), bitcoind needs some too !
|
||||
# ulimit -n 1000000 # Can't be $(ulimit -Hn), bitcoind needs some too !
|
||||
|
||||
# Needed because the datasets tree is too big lol
|
||||
echo "Increasing stack size..."
|
||||
@@ -1,22 +0,0 @@
|
||||
[package]
|
||||
name = "server"
|
||||
version = "0.5.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
axum = "0.7.7"
|
||||
bincode = { git = "https://github.com/bincode-org/bincode.git" }
|
||||
chrono = "0.4.38"
|
||||
color-eyre = "0.6.3"
|
||||
derive_deref = "1.1.1"
|
||||
itertools = "0.13.0"
|
||||
parser = { path = "../parser" }
|
||||
regex = "1.11.0"
|
||||
reqwest = { version = "0.12.8", features = ["json"] }
|
||||
serde = { version = "1.0.210", features = ["derive"] }
|
||||
serde_json = { version = "1.0.128" }
|
||||
swc = "0.289.1"
|
||||
swc_common = "0.40.1"
|
||||
tokio = { version = "1.40.0", features = ["full"] }
|
||||
tower-http = { version = "0.6.1", features = ["compression-full"] }
|
||||
# oxc = { version = "0.34.0", features = ["codegen", "minifier"] }
|
||||
@@ -1,4 +0,0 @@
|
||||
sudo launchctl stop com.cloudflare.cloudflared
|
||||
sudo launchctl unload /Library/LaunchDaemons/com.cloudflare.cloudflared.plist
|
||||
sudo launchctl load /Library/LaunchDaemons/com.cloudflare.cloudflared.plist
|
||||
sudo launchctl start com.cloudflare.cloudflared
|
||||
@@ -1,14 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
if cargo watch --help &> /dev/null; then
|
||||
TRIGGER="./in/datasets_len.txt"
|
||||
|
||||
if [ ! -f "$TRIGGER" ]; then
|
||||
mkdir "./in"
|
||||
echo "0" > $TRIGGER
|
||||
fi
|
||||
|
||||
cargo watch --no-vcs-ignores -w "./src" -w "$TRIGGER" -x "run -r"
|
||||
else
|
||||
cargo run -r
|
||||
fi
|
||||
@@ -1,157 +0,0 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use itertools::Itertools;
|
||||
use parser::{Json, Serialization};
|
||||
|
||||
use crate::Grouped;
|
||||
|
||||
use super::Paths;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Route {
|
||||
pub url_path: String,
|
||||
pub file_path: PathBuf,
|
||||
pub values_type: String,
|
||||
pub serialization: Serialization,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deref, DerefMut)]
|
||||
pub struct Routes(pub Grouped<HashMap<String, Route>>);
|
||||
|
||||
const INPUTS_PATH: &str = "./in";
|
||||
const WEBSITE_TYPES_PATH: &str = "../website/scripts/types";
|
||||
|
||||
impl Routes {
|
||||
pub fn build() -> Self {
|
||||
let path_to_type: BTreeMap<String, String> =
|
||||
Json::import(Path::new(&format!("{INPUTS_PATH}/disk_path_to_type.json"))).unwrap();
|
||||
|
||||
let mut routes = Routes::default();
|
||||
|
||||
path_to_type.into_iter().for_each(|(key, value)| {
|
||||
let mut split_key = key.split('/').collect_vec();
|
||||
let last = split_key.pop().unwrap().to_owned();
|
||||
|
||||
let mut skip = 2;
|
||||
|
||||
let mut serialization = Serialization::Binary;
|
||||
|
||||
if *split_key.get(1).unwrap() == "price" {
|
||||
skip = 1;
|
||||
serialization = Serialization::Json;
|
||||
}
|
||||
|
||||
let mut split_key = split_key.iter().skip(skip).collect_vec();
|
||||
|
||||
// Use case for: "../datasets/last": "Value",
|
||||
if split_key.is_empty() {
|
||||
split_key.push(&"last");
|
||||
}
|
||||
|
||||
let map_key = split_key.iter().join("_");
|
||||
|
||||
let url_path = split_key.iter().join("-");
|
||||
|
||||
let file_path = PathBuf::from(key.to_owned());
|
||||
let values_type = value.to_owned();
|
||||
|
||||
if last == "date" {
|
||||
routes.date.insert(
|
||||
map_key,
|
||||
Route {
|
||||
url_path: format!("date-to-{url_path}"),
|
||||
file_path,
|
||||
values_type,
|
||||
serialization,
|
||||
},
|
||||
);
|
||||
} else if last == "height" {
|
||||
routes.height.insert(
|
||||
map_key,
|
||||
Route {
|
||||
url_path: format!("height-to-{url_path}"),
|
||||
file_path,
|
||||
values_type,
|
||||
serialization,
|
||||
},
|
||||
);
|
||||
} else if last == "last" {
|
||||
routes.last.insert(
|
||||
map_key,
|
||||
Route {
|
||||
url_path,
|
||||
file_path,
|
||||
values_type,
|
||||
serialization,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
dbg!(&key, value, &last);
|
||||
panic!("")
|
||||
}
|
||||
});
|
||||
|
||||
routes
|
||||
}
|
||||
|
||||
pub fn generate_dts_file(&self) {
|
||||
let map_to_type = |name: &str, map: &HashMap<String, Route>| -> String {
|
||||
let paths = map
|
||||
.values()
|
||||
.map(|route| format!("\"{}\"", route.url_path))
|
||||
.join(" | ");
|
||||
|
||||
format!("export type {}Path = {};\n", name, paths)
|
||||
};
|
||||
|
||||
let date_type = map_to_type("Date", &self.date);
|
||||
|
||||
let height_type = map_to_type("Height", &self.height);
|
||||
|
||||
let last_type = map_to_type("Last", &self.last);
|
||||
|
||||
fs::write(
|
||||
format!("{WEBSITE_TYPES_PATH}/paths.d.ts"),
|
||||
format!("// This file is auto generated by the server\n// Manual changes are forbidden\n\n{date_type}\n{height_type}\n{last_type}"),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn to_full_paths(&self, host: String) -> Paths {
|
||||
let url = {
|
||||
let scheme = if host.contains("0.0.0.0") || host.contains("localhost") {
|
||||
"http"
|
||||
} else {
|
||||
"https"
|
||||
};
|
||||
|
||||
format!("{scheme}://{host}")
|
||||
};
|
||||
|
||||
let transform = |map: &HashMap<String, Route>| -> BTreeMap<String, String> {
|
||||
map.iter()
|
||||
.map(|(key, route)| {
|
||||
(
|
||||
key.to_owned(),
|
||||
format!("{url}/api/{}", route.url_path.to_owned()),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
let date_paths = transform(&self.date);
|
||||
let height_paths = transform(&self.height);
|
||||
let last_paths = transform(&self.last);
|
||||
|
||||
Paths(Grouped {
|
||||
date: date_paths,
|
||||
height: height_paths,
|
||||
last: last_paths,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,7 @@
|
||||
mod binary;
|
||||
mod consts;
|
||||
mod json;
|
||||
mod serialization;
|
||||
|
||||
pub use binary::*;
|
||||
pub use consts::*;
|
||||
pub use json::*;
|
||||
pub use serialization::*;
|
||||
@@ -1,4 +1,8 @@
|
||||
use std::{fmt::Debug, fs, path::Path};
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use bincode::{Decode, Encode};
|
||||
@@ -27,17 +31,6 @@ impl Serialization {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_path(path: &Path) -> Self {
|
||||
let path = path.to_str().unwrap();
|
||||
if path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION) {
|
||||
Self::Binary
|
||||
} else if path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION) {
|
||||
Self::Json
|
||||
} else {
|
||||
panic!("Extension \"{path}\" isn't supported")
|
||||
}
|
||||
}
|
||||
|
||||
pub fn import<T>(&self, path: &Path) -> color_eyre::Result<T>
|
||||
where
|
||||
T: Debug + DeserializeOwned + Decode,
|
||||
@@ -119,3 +112,17 @@ impl Serialization {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&PathBuf> for Serialization {
|
||||
type Error = ();
|
||||
|
||||
fn try_from(path: &PathBuf) -> Result<Self, Self::Error> {
|
||||
let extension = path.extension().ok_or(())?.to_str().unwrap();
|
||||
|
||||
match extension {
|
||||
BIN_EXTENSION | COMPRESSED_BIN_EXTENSION => Ok(Self::Binary),
|
||||
JSON_EXTENSION => Ok(Self::Json),
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
use biter::bitcoincore_rpc::Client;
|
||||
use log::info;
|
||||
use rlimit::{getrlimit, setrlimit, Resource};
|
||||
|
||||
mod io;
|
||||
mod parser;
|
||||
mod server;
|
||||
mod structs;
|
||||
mod utils;
|
||||
|
||||
use parser::{Databases, Datasets};
|
||||
use structs::{Config, Exit};
|
||||
use utils::init_log;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
init_log();
|
||||
|
||||
let (_, nofile_limit) = getrlimit(Resource::NOFILE).unwrap();
|
||||
setrlimit(Resource::NOFILE, 138_240, nofile_limit)?;
|
||||
|
||||
std::thread::Builder::new()
|
||||
.stack_size(getrlimit(Resource::STACK).unwrap().1 as usize)
|
||||
.spawn(|| -> color_eyre::Result<()> {
|
||||
let exit = Exit::new();
|
||||
|
||||
let config = Config::import()?;
|
||||
|
||||
info!("Starting...");
|
||||
|
||||
let rpc = Client::from(&config);
|
||||
|
||||
let databases = Databases::import(&config);
|
||||
|
||||
let datasets = Datasets::import(&config)?;
|
||||
|
||||
let paths_to_type = datasets.get_paths_to_type(&config);
|
||||
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
let config_clone = config.clone();
|
||||
let run_parser = config.parser();
|
||||
let run_server = config.server();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
if run_server {
|
||||
server::main(paths_to_type, &config_clone).await.unwrap();
|
||||
} else {
|
||||
info!("Skipping server");
|
||||
}
|
||||
});
|
||||
|
||||
if run_parser {
|
||||
parser::main(&config, &rpc, &exit, databases, datasets)?;
|
||||
} else {
|
||||
info!("Skipping parser");
|
||||
}
|
||||
|
||||
handle.await?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
})?
|
||||
.join()
|
||||
.unwrap()
|
||||
}
|
||||
@@ -1,52 +1,62 @@
|
||||
use std::thread::{self};
|
||||
|
||||
use log::info;
|
||||
|
||||
use crate::{
|
||||
databases::Databases,
|
||||
datasets::AllDatasets,
|
||||
states::States,
|
||||
structs::{Date, Height},
|
||||
utils::{log, time},
|
||||
Exit,
|
||||
parser::{databases::Databases, datasets::Datasets, states::States},
|
||||
structs::{Config, Date, Exit, Height},
|
||||
utils::time,
|
||||
};
|
||||
|
||||
pub struct ExportedData<'a> {
|
||||
pub config: &'a Config,
|
||||
pub databases: Option<&'a mut Databases>,
|
||||
pub datasets: &'a mut AllDatasets,
|
||||
pub datasets: &'a mut Datasets,
|
||||
pub date: Date,
|
||||
pub defragment: bool,
|
||||
pub exit: Exit,
|
||||
pub height: Height,
|
||||
pub states: Option<&'a States>,
|
||||
pub exit: Exit,
|
||||
}
|
||||
|
||||
pub fn export(
|
||||
ExportedData {
|
||||
config,
|
||||
databases,
|
||||
datasets,
|
||||
states,
|
||||
height,
|
||||
date,
|
||||
defragment,
|
||||
exit,
|
||||
height,
|
||||
states,
|
||||
}: ExportedData,
|
||||
) -> color_eyre::Result<()> {
|
||||
if exit.active() {
|
||||
log("Exit in progress, skipping export");
|
||||
info!("Exit in progress, skipping export");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
exit.block();
|
||||
|
||||
log("Exporting...");
|
||||
info!("Exporting...");
|
||||
if defragment {
|
||||
info!("Will also defragment databases, please be patient it might take a while")
|
||||
}
|
||||
|
||||
time("Total save time", || -> color_eyre::Result<()> {
|
||||
time("Datasets saved", || datasets.export())?;
|
||||
time("Datasets saved", || datasets.export(config))?;
|
||||
|
||||
thread::scope(|s| {
|
||||
if let Some(databases) = databases {
|
||||
s.spawn(|| time("Databases saved", || databases.export(height, date)));
|
||||
s.spawn(|| {
|
||||
time("Databases saved", || {
|
||||
databases.export(height, date, defragment)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(states) = states {
|
||||
s.spawn(|| time("States saved", || states.export()));
|
||||
s.spawn(|| time("States saved", || states.export(config)));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,63 +1,52 @@
|
||||
use std::{collections::BTreeSet, time::Instant};
|
||||
|
||||
use biter::bitcoincore_rpc::Client;
|
||||
use chrono::Datelike;
|
||||
use export::ExportedData;
|
||||
use itertools::Itertools;
|
||||
|
||||
use log::info;
|
||||
use parse::ParseData;
|
||||
|
||||
use crate::{
|
||||
actions::{export, find_first_inserted_unsafe_height, parse},
|
||||
create_rpc,
|
||||
databases::Databases,
|
||||
datasets::{AllDatasets, ComputeData},
|
||||
io::OUTPUTS_FOLDER_PATH,
|
||||
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
|
||||
structs::{DateData, MapKey, Timestamp},
|
||||
utils::{generate_allocation_files, log, time},
|
||||
Config, Exit, Height,
|
||||
parser::{
|
||||
actions::{export, find_first_inserted_unsafe_height, parse},
|
||||
databases::Databases,
|
||||
datasets::{ComputeData, Datasets},
|
||||
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
|
||||
},
|
||||
structs::{Config, DateData, Exit, Height, MapKey, Timestamp},
|
||||
utils::{generate_allocation_files, time},
|
||||
};
|
||||
|
||||
pub fn iter_blocks(
|
||||
config: &mut Config,
|
||||
config: &Config,
|
||||
rpc: &biter::bitcoincore_rpc::Client,
|
||||
approx_block_count: usize,
|
||||
exit: Exit,
|
||||
databases: &mut Databases,
|
||||
datasets: &mut Datasets,
|
||||
) -> color_eyre::Result<()> {
|
||||
log("Starting...");
|
||||
let mut states = States::import(config).unwrap_or_default();
|
||||
|
||||
let mut datasets = AllDatasets::import(config)?;
|
||||
|
||||
log("Imported datasets");
|
||||
|
||||
let mut databases = Databases::import();
|
||||
|
||||
if config.first_defragment() {
|
||||
databases.defragment(&exit);
|
||||
config.disable_defragment();
|
||||
}
|
||||
|
||||
log("Imported databases");
|
||||
|
||||
let mut states = States::import().unwrap_or_default();
|
||||
|
||||
log("Imported states");
|
||||
info!("Imported states");
|
||||
|
||||
let first_unsafe_heights =
|
||||
find_first_inserted_unsafe_height(&mut states, &mut databases, &mut datasets);
|
||||
find_first_inserted_unsafe_height(&mut states, databases, datasets, config);
|
||||
|
||||
let mut height = first_unsafe_heights.min();
|
||||
|
||||
log(&format!("Starting parsing at height: {height}"));
|
||||
info!("Starting parsing at height: {height}");
|
||||
|
||||
let mut next_block_opt = None;
|
||||
let mut blocks_loop_date = None;
|
||||
let mut next_date_opt;
|
||||
|
||||
let block_receiver = biter::new(
|
||||
config.datadir.as_ref().unwrap(),
|
||||
OUTPUTS_FOLDER_PATH,
|
||||
&config.path_bitcoindir(),
|
||||
Some(height.to_usize()),
|
||||
None,
|
||||
create_rpc(config).unwrap(),
|
||||
Client::from(config),
|
||||
);
|
||||
|
||||
let mut block_iter = block_receiver.iter();
|
||||
@@ -93,15 +82,13 @@ pub fn iter_blocks(
|
||||
panic!()
|
||||
}
|
||||
|
||||
let next_block_date = next_block_opt.as_ref().map(|(_, next_block, _)| {
|
||||
next_date_opt = next_block_opt.as_ref().map(|(_, next_block, _)| {
|
||||
Timestamp::wrap(next_block.header.time).to_date()
|
||||
});
|
||||
|
||||
// Always run for the first block of the loop
|
||||
if blocks_loop_date.is_none() {
|
||||
log(&format!(
|
||||
"Processing {current_block_date} (height: {height})..."
|
||||
));
|
||||
info!("Processing {current_block_date} (height: {height})...");
|
||||
|
||||
blocks_loop_date.replace(current_block_date);
|
||||
|
||||
@@ -125,7 +112,7 @@ pub fn iter_blocks(
|
||||
panic!("current block should always have the same date as the current blocks loop");
|
||||
}
|
||||
|
||||
let is_date_last_block = next_block_date
|
||||
let is_date_last_block = next_date_opt
|
||||
// Do NOT change `blocks_loop_date` to `current_block_date` !!!
|
||||
.map_or(true, |next_block_date| blocks_loop_date < next_block_date);
|
||||
|
||||
@@ -159,16 +146,17 @@ pub fn iter_blocks(
|
||||
}
|
||||
|
||||
parse(ParseData {
|
||||
rpc,
|
||||
block: current_block,
|
||||
block_index: blocks_loop_i,
|
||||
compute_addresses,
|
||||
databases: &mut databases,
|
||||
datasets: &mut datasets,
|
||||
config,
|
||||
databases,
|
||||
datasets,
|
||||
date: blocks_loop_date,
|
||||
first_date_height: height,
|
||||
height: current_block_height,
|
||||
is_date_last_block,
|
||||
rpc,
|
||||
states: &mut states,
|
||||
});
|
||||
}
|
||||
@@ -178,7 +166,7 @@ pub fn iter_blocks(
|
||||
if is_date_last_block {
|
||||
height += blocks_loop_i;
|
||||
|
||||
let is_check_point = next_block_date
|
||||
let is_check_point = next_date_opt
|
||||
.as_ref()
|
||||
.map_or(true, |date| date.is_first_of_month());
|
||||
|
||||
@@ -201,13 +189,13 @@ pub fn iter_blocks(
|
||||
// Don't remember why -1
|
||||
let last_height = height - 1_u32;
|
||||
|
||||
log(&format!(
|
||||
"Parsing group took {} seconds (last height: {last_height})\n",
|
||||
info!(
|
||||
"Parsing group took {} seconds (last height: {last_height})",
|
||||
instant.elapsed().as_secs_f32(),
|
||||
));
|
||||
);
|
||||
|
||||
if first_unsafe_heights.computed <= last_height {
|
||||
log("Computing datasets...");
|
||||
info!("Computing datasets...");
|
||||
time("Computing datasets", || {
|
||||
let dates = processed_dates.into_iter().collect_vec();
|
||||
|
||||
@@ -223,10 +211,17 @@ pub fn iter_blocks(
|
||||
if !config.dry_run() {
|
||||
let is_safe = height.is_safe(approx_block_count);
|
||||
|
||||
let defragment = is_safe
|
||||
&& next_date_opt.is_some_and(|date| {
|
||||
date.year() >= 2020 && date.is_january() && date.is_first_of_month()
|
||||
});
|
||||
|
||||
export(ExportedData {
|
||||
databases: is_safe.then_some(&mut databases),
|
||||
datasets: &mut datasets,
|
||||
config,
|
||||
databases: is_safe.then_some(databases),
|
||||
datasets,
|
||||
date: blocks_loop_date.unwrap(),
|
||||
defragment,
|
||||
height: last_height,
|
||||
states: is_safe.then_some(&states),
|
||||
exit: exit.clone(),
|
||||
@@ -234,11 +229,11 @@ pub fn iter_blocks(
|
||||
|
||||
if config.record_ram_usage() {
|
||||
time("Exporing allocation files", || {
|
||||
generate_allocation_files(&datasets, &databases, &states, last_height)
|
||||
generate_allocation_files(datasets, databases, &states, last_height)
|
||||
})?;
|
||||
}
|
||||
} else {
|
||||
log("Skipping export");
|
||||
info!("Skipping export");
|
||||
}
|
||||
|
||||
println!();
|
||||
@@ -1,9 +1,12 @@
|
||||
use log::info;
|
||||
|
||||
use crate::{
|
||||
databases::Databases,
|
||||
datasets::{AllDatasets, AnyDatasets},
|
||||
states::States,
|
||||
structs::Height,
|
||||
utils::log,
|
||||
parser::{
|
||||
databases::Databases,
|
||||
datasets::{AnyDatasets, Datasets},
|
||||
states::States,
|
||||
},
|
||||
structs::{Config, Height},
|
||||
};
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
@@ -21,7 +24,8 @@ impl Heights {
|
||||
pub fn find_first_inserted_unsafe_height(
|
||||
states: &mut States,
|
||||
databases: &mut Databases,
|
||||
datasets: &mut AllDatasets,
|
||||
datasets: &mut Datasets,
|
||||
config: &Config,
|
||||
) -> Heights {
|
||||
let min_initial_inserted_last_address_height = datasets
|
||||
.address
|
||||
@@ -51,7 +55,7 @@ pub fn find_first_inserted_unsafe_height(
|
||||
.map(|date_data| date_data.date)
|
||||
.and_then(|last_safe_date| {
|
||||
if !usable_databases {
|
||||
log("Unusable databases");
|
||||
info!("Unusable databases");
|
||||
|
||||
return None;
|
||||
}
|
||||
@@ -61,8 +65,8 @@ pub fn find_first_inserted_unsafe_height(
|
||||
let min_datasets_inserted_last_height = datasets_min_initial_states.inserted.last_height;
|
||||
let min_datasets_inserted_last_date = datasets_min_initial_states.inserted.last_date;
|
||||
|
||||
log(&format!("min_datasets_inserted_last_height: {:?}", min_datasets_inserted_last_height));
|
||||
log(&format!("min_datasets_inserted_last_date: {:?}", min_datasets_inserted_last_date));
|
||||
info!("min_datasets_inserted_last_height: {:?}", min_datasets_inserted_last_height);
|
||||
info!("min_datasets_inserted_last_date: {:?}", min_datasets_inserted_last_date);
|
||||
|
||||
let inserted_last_date_is_older_than_saved_state = min_datasets_inserted_last_date.map_or(true, |min_datasets_last_date| min_datasets_last_date < last_safe_date);
|
||||
|
||||
@@ -80,7 +84,7 @@ pub fn find_first_inserted_unsafe_height(
|
||||
let inserted_heights_and_dates_are_out_of_sync = min_datasets_inserted_last_height.map_or(true, |min_datasets_inserted_last_height| min_datasets_inserted_last_height < last_safe_height);
|
||||
|
||||
if inserted_heights_and_dates_are_out_of_sync {
|
||||
log(&format!("last_safe_height ({last_safe_height}) > min_datasets_height ({min_datasets_inserted_last_height:?})"));
|
||||
info!("last_safe_height ({last_safe_height}) > min_datasets_height ({min_datasets_inserted_last_height:?})");
|
||||
|
||||
None
|
||||
} else {
|
||||
@@ -108,7 +112,7 @@ pub fn find_first_inserted_unsafe_height(
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
log("Starting over...");
|
||||
info!("Starting over...");
|
||||
|
||||
let include_addresses = !usable_databases
|
||||
|| min_initial_inserted_last_address_date.is_none()
|
||||
@@ -119,7 +123,7 @@ pub fn find_first_inserted_unsafe_height(
|
||||
// panic!("");
|
||||
// }
|
||||
|
||||
states.reset(include_addresses);
|
||||
states.reset(config, include_addresses);
|
||||
|
||||
databases.reset(include_addresses);
|
||||
|
||||
@@ -9,18 +9,21 @@ use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
databases::{
|
||||
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex,
|
||||
Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount,
|
||||
},
|
||||
datasets::{AllDatasets, InsertData},
|
||||
states::{
|
||||
AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates,
|
||||
States, UTXOCohortsOneShotStates, UTXOCohortsSentStates,
|
||||
parser::{
|
||||
databases::{
|
||||
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex,
|
||||
Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount,
|
||||
},
|
||||
datasets::{Datasets, InsertData},
|
||||
states::{
|
||||
AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates,
|
||||
States, UTXOCohortsOneShotStates, UTXOCohortsSentStates,
|
||||
},
|
||||
},
|
||||
structs::{
|
||||
Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Counter, Date,
|
||||
EmptyAddressData, Height, PartialTxoutData, Price, SentData, Timestamp, TxData, TxoutIndex,
|
||||
Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Config, Counter,
|
||||
Date, EmptyAddressData, Height, PartialTxoutData, Price, SentData, Timestamp, TxData,
|
||||
TxoutIndex,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -28,9 +31,10 @@ pub struct ParseData<'a> {
|
||||
// pub bitcoin_cli: &'a BitcoinCli,
|
||||
pub block: Block,
|
||||
pub block_index: usize,
|
||||
pub config: &'a Config,
|
||||
pub compute_addresses: bool,
|
||||
pub databases: &'a mut Databases,
|
||||
pub datasets: &'a mut AllDatasets,
|
||||
pub datasets: &'a mut Datasets,
|
||||
pub date: Date,
|
||||
pub first_date_height: Height,
|
||||
pub height: Height,
|
||||
@@ -43,6 +47,7 @@ pub fn parse(
|
||||
ParseData {
|
||||
block,
|
||||
block_index,
|
||||
config,
|
||||
compute_addresses,
|
||||
databases,
|
||||
datasets,
|
||||
@@ -72,7 +77,7 @@ pub fn parse(
|
||||
let block_price = Price::from_dollar(
|
||||
datasets
|
||||
.price
|
||||
.get_height_ohlc(height, timestamp, previous_timestamp)
|
||||
.get_height_ohlc(height, timestamp, previous_timestamp, config)
|
||||
.unwrap_or_else(|_| panic!("Expect {height} to have a price"))
|
||||
.close as f64,
|
||||
);
|
||||
@@ -1,35 +1,35 @@
|
||||
// https://docs.rs/sanakirja/latest/sanakirja/index.html
|
||||
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt::Debug,
|
||||
fs, mem,
|
||||
fs, io, mem,
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
|
||||
// https://docs.rs/sanakirja/latest/sanakirja/index.html
|
||||
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
|
||||
//
|
||||
// Seems indeed much faster than ReDB and LMDB (heed)
|
||||
// But a lot has changed code wise between them so a retest wouldn't hurt
|
||||
//
|
||||
// Possible compression: https://pijul.org/posts/sanakirja-zstd/
|
||||
use sanakirja::{
|
||||
btree::{self, page, Db_, Iter},
|
||||
Commit, Env, Error, MutTxn, RootDb, Storable,
|
||||
};
|
||||
|
||||
///
|
||||
/// Simple wrapper around Sanakirja Database with cached puts and dels for safe use outside exports.
|
||||
///
|
||||
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching "gets" along the way.
|
||||
///
|
||||
#[derive(Allocative)]
|
||||
#[allocative(bound = "Key: Allocative, Value: Allocative")]
|
||||
/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way.
|
||||
pub struct Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq,
|
||||
{
|
||||
pub cached_puts: BTreeMap<Key, Value>,
|
||||
pub cached_dels: BTreeSet<Key>,
|
||||
path: PathBuf,
|
||||
cached_puts: BTreeMap<Key, Value>,
|
||||
cached_dels: BTreeSet<Key>,
|
||||
#[allocative(skip)]
|
||||
db: Db_<Key, Value, page::Page<Key, Value>>,
|
||||
#[allocative(skip)]
|
||||
@@ -62,11 +62,12 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn iter(&self) -> Iter<'_, MutTxn<Env, ()>, Key, Value, page::Page<Key, Value>> {
|
||||
btree::iter(&self.txn, &self.db, None).unwrap()
|
||||
}
|
||||
|
||||
pub fn iter_collect(&self) -> BTreeMap<Key, Value>
|
||||
pub fn collect(&self) -> BTreeMap<Key, Value>
|
||||
where
|
||||
Value: Clone,
|
||||
{
|
||||
@@ -76,6 +77,7 @@ where
|
||||
.collect::<_>()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get(&self, key: &Key) -> Option<&Value> {
|
||||
if let Some(cached_put) = self.get_from_puts(key) {
|
||||
return Some(cached_put);
|
||||
@@ -84,6 +86,7 @@ where
|
||||
self.db_get(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn db_get(&self, key: &Key) -> Option<&Value> {
|
||||
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
|
||||
|
||||
@@ -96,17 +99,17 @@ where
|
||||
None
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
#[inline]
|
||||
pub fn get_from_puts(&self, key: &Key) -> Option<&Value> {
|
||||
self.cached_puts.get(key)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
#[inline]
|
||||
pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> {
|
||||
self.cached_puts.get_mut(key)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
#[inline]
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.remove_from_puts(key).or_else(|| {
|
||||
self.db_remove(key);
|
||||
@@ -139,7 +142,6 @@ where
|
||||
#[inline]
|
||||
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.cached_dels.remove(&key);
|
||||
|
||||
self.unsafe_insert(key, value)
|
||||
}
|
||||
|
||||
@@ -165,12 +167,11 @@ where
|
||||
}
|
||||
|
||||
pub trait AnyDatabase {
|
||||
fn export(self) -> color_eyre::Result<(), Error>;
|
||||
fn boxed_export(self: Box<Self>) -> color_eyre::Result<(), Error>;
|
||||
#[allow(unused)]
|
||||
fn defragment(self);
|
||||
fn boxed_defragment(self: Box<Self>);
|
||||
fn destroy(self);
|
||||
fn export(self, defragment: bool) -> color_eyre::Result<(), Error>;
|
||||
fn boxed_export(self: Box<Self>, defragment: bool) -> color_eyre::Result<(), Error>;
|
||||
#[allow(unused)]
|
||||
fn destroy(self) -> io::Result<()>;
|
||||
}
|
||||
|
||||
impl<Key, Value> AnyDatabase for Database<Key, Value>
|
||||
@@ -178,11 +179,31 @@ where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq + Clone,
|
||||
{
|
||||
fn export(self) -> color_eyre::Result<(), Error> {
|
||||
Box::new(self).boxed_export()
|
||||
fn export(self, defragment: bool) -> color_eyre::Result<(), Error> {
|
||||
Box::new(self).boxed_export(defragment)
|
||||
}
|
||||
|
||||
fn boxed_export(mut self: Box<Self>) -> color_eyre::Result<(), Error> {
|
||||
fn boxed_export(mut self: Box<Self>, defragment: bool) -> color_eyre::Result<(), Error> {
|
||||
if defragment {
|
||||
let mut btree = self.as_ref().collect();
|
||||
|
||||
let path = self.path.to_owned();
|
||||
self.cached_dels.iter().for_each(|key| {
|
||||
btree.remove(key);
|
||||
});
|
||||
btree.append(&mut self.cached_puts);
|
||||
|
||||
self.destroy()?;
|
||||
|
||||
*self = Self::open(path).unwrap();
|
||||
|
||||
if !self.is_empty() {
|
||||
panic!()
|
||||
}
|
||||
|
||||
self.cached_puts = btree;
|
||||
}
|
||||
|
||||
if self.cached_dels.is_empty() && self.cached_puts.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -198,35 +219,11 @@ where
|
||||
self.txn.commit()
|
||||
}
|
||||
|
||||
fn defragment(self) {
|
||||
Box::new(self).boxed_defragment()
|
||||
}
|
||||
|
||||
fn boxed_defragment(self: Box<Self>) {
|
||||
let btree = self.iter_collect();
|
||||
|
||||
let path = self.path.to_owned();
|
||||
|
||||
self.destroy();
|
||||
|
||||
let mut db = Self::open(path).unwrap();
|
||||
|
||||
if !db.is_empty() {
|
||||
panic!()
|
||||
}
|
||||
|
||||
db.cached_puts = btree;
|
||||
db.export().unwrap();
|
||||
}
|
||||
|
||||
fn destroy(self) {
|
||||
fn destroy(self) -> io::Result<()> {
|
||||
let path = self.path.to_owned();
|
||||
|
||||
drop(self);
|
||||
|
||||
fs::remove_file(&path).unwrap_or_else(|_| {
|
||||
dbg!(path);
|
||||
panic!("Error");
|
||||
});
|
||||
fs::remove_file(&path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
use std::{fs, io, path::Path};
|
||||
|
||||
use log::info;
|
||||
|
||||
use crate::structs::{Config, Date, Height};
|
||||
|
||||
use super::{AnyDatabase, Metadata};
|
||||
|
||||
pub trait AnyDatabaseGroup
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
fn init(config: &Config) -> Self {
|
||||
let s = Self::import(config);
|
||||
s.create_dir_all().unwrap();
|
||||
s
|
||||
}
|
||||
|
||||
fn import(config: &Config) -> Self;
|
||||
|
||||
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>>;
|
||||
fn open_all(&mut self);
|
||||
|
||||
fn metadata(&mut self) -> &mut Metadata;
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata().export(height, date)
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(self.path())
|
||||
}
|
||||
|
||||
fn remove_dir_all(&self) -> color_eyre::Result<(), io::Error> {
|
||||
fs::remove_dir_all(self.path())
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
|
||||
info!(
|
||||
"Reset {}",
|
||||
self.path()
|
||||
.components()
|
||||
.last()
|
||||
.unwrap()
|
||||
.as_os_str()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
);
|
||||
|
||||
self.reset_metadata();
|
||||
self.remove_dir_all()?;
|
||||
self.create_dir_all()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self);
|
||||
|
||||
fn path(&self) -> &Path;
|
||||
}
|
||||
+19
-20
@@ -2,6 +2,7 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
@@ -9,8 +10,8 @@ use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
states::AddressCohortsDurableStates,
|
||||
structs::{AddressData, Date, Height},
|
||||
parser::states::AddressCohortsDurableStates,
|
||||
structs::{AddressData, Config},
|
||||
utils::time,
|
||||
};
|
||||
|
||||
@@ -22,8 +23,8 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AddressIndexToAddressData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
pub map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -72,6 +73,7 @@ impl AddressIndexToAddressData {
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path().to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -80,7 +82,7 @@ impl AddressIndexToAddressData {
|
||||
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
@@ -113,30 +115,23 @@ impl AddressIndexToAddressData {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for AddressIndexToAddressData {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config
|
||||
.path_databases()
|
||||
.join("address_index_to_address_data");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"address_index_to_address_data"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -167,7 +162,11 @@ impl AnyDatabaseGroup for AddressIndexToAddressData {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+18
-19
@@ -2,12 +2,13 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::structs::{Date, EmptyAddressData, Height};
|
||||
use crate::structs::{Config, EmptyAddressData};
|
||||
|
||||
use super::{
|
||||
AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE,
|
||||
@@ -19,8 +20,8 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AddressIndexToEmptyAddressData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -72,6 +73,7 @@ impl AddressIndexToEmptyAddressData {
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path.to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -80,7 +82,7 @@ impl AddressIndexToEmptyAddressData {
|
||||
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
@@ -92,30 +94,23 @@ impl AddressIndexToEmptyAddressData {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config
|
||||
.path_databases()
|
||||
.join("address_index_to_empty_address_data");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"address_index_to_empty_address_data"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -146,7 +141,11 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+47
-44
@@ -7,7 +7,7 @@ use std::{
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::structs::{Address, Date, Height, U8x19, U8x31};
|
||||
use crate::structs::{Address, Config, U8x19, U8x31};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata};
|
||||
|
||||
@@ -30,6 +30,7 @@ type MultisigDatabase = U32Database;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AddressToAddressIndex {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
p2pk: BTreeMap<u16, P2PKDatabase>,
|
||||
@@ -160,12 +161,12 @@ impl AddressToAddressIndex {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn path_p2pk() -> PathBuf {
|
||||
Self::root().join("p2pk")
|
||||
fn path_p2pk(&self) -> PathBuf {
|
||||
self.path().join("p2pk")
|
||||
}
|
||||
|
||||
pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase {
|
||||
let path = Self::path_p2pk();
|
||||
let path = self.path_p2pk();
|
||||
self.p2pk.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
Database::open(path).unwrap()
|
||||
@@ -173,7 +174,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2pk(&mut self) {
|
||||
let path = Self::path_p2pk();
|
||||
let path = self.path_p2pk();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -184,12 +185,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2pkh() -> PathBuf {
|
||||
Self::root().join("p2pkh")
|
||||
fn path_p2pkh(&self) -> PathBuf {
|
||||
self.path().join("p2pkh")
|
||||
}
|
||||
|
||||
pub fn open_p2pkh(&mut self, prefix: u16) -> &mut P2PKHDatabase {
|
||||
let path = Self::path_p2pkh();
|
||||
let path = self.path_p2pkh();
|
||||
|
||||
self.p2pkh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -198,7 +199,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2pkh(&mut self) {
|
||||
let path = Self::path_p2pkh();
|
||||
let path = self.path_p2pkh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -209,12 +210,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2sh() -> PathBuf {
|
||||
Self::root().join("p2sh")
|
||||
fn path_p2sh(&self) -> PathBuf {
|
||||
self.path().join("p2sh")
|
||||
}
|
||||
|
||||
pub fn open_p2sh(&mut self, prefix: u16) -> &mut P2SHDatabase {
|
||||
let path = Self::path_p2sh();
|
||||
let path = self.path_p2sh();
|
||||
|
||||
self.p2sh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -223,7 +224,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2sh(&mut self) {
|
||||
let path = Self::path_p2sh();
|
||||
let path = self.path_p2sh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -234,12 +235,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2wpkh() -> PathBuf {
|
||||
Self::root().join("p2wpkh")
|
||||
fn path_p2wpkh(&self) -> PathBuf {
|
||||
self.path().join("p2wpkh")
|
||||
}
|
||||
|
||||
pub fn open_p2wpkh(&mut self, prefix: u16) -> &mut P2WPKHDatabase {
|
||||
let path = Self::path_p2wpkh();
|
||||
let path = self.path_p2wpkh();
|
||||
|
||||
self.p2wpkh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -248,7 +249,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2wpkh(&mut self) {
|
||||
let path = Self::path_p2wpkh();
|
||||
let path = self.path_p2wpkh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -259,12 +260,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2wsh() -> PathBuf {
|
||||
Self::root().join("p2wsh")
|
||||
fn path_p2wsh(&self) -> PathBuf {
|
||||
self.path().join("p2wsh")
|
||||
}
|
||||
|
||||
pub fn open_p2wsh(&mut self, prefix: u16) -> &mut P2WSHDatabase {
|
||||
let path = Self::path_p2wsh();
|
||||
let path = self.path_p2wsh();
|
||||
|
||||
self.p2wsh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -273,7 +274,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2wsh(&mut self) {
|
||||
let path = Self::path_p2wsh();
|
||||
let path = self.path_p2wsh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -284,12 +285,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2tr() -> PathBuf {
|
||||
Self::root().join("p2tr")
|
||||
fn path_p2tr(&self) -> PathBuf {
|
||||
self.path().join("p2tr")
|
||||
}
|
||||
|
||||
pub fn open_p2tr(&mut self, prefix: u16) -> &mut P2TRDatabase {
|
||||
let path = Self::path_p2tr();
|
||||
let path = self.path_p2tr();
|
||||
|
||||
self.p2tr.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -298,7 +299,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2tr(&mut self) {
|
||||
let path = Self::path_p2tr();
|
||||
let path = self.path_p2tr();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -311,34 +312,36 @@ impl AddressToAddressIndex {
|
||||
|
||||
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
|
||||
self.unknown
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("unknown")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("unknown")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
|
||||
self.op_return
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("op_return")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("op_return")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
|
||||
self.push_only
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("push_only")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("push_only")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_empty(&mut self) -> &mut UnknownDatabase {
|
||||
self.empty
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("empty")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("empty")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
|
||||
self.multisig
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("multisig")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("multisig")).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for AddressToAddressIndex {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("address_to_address_index");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
|
||||
p2pk: BTreeMap::default(),
|
||||
p2pkh: BTreeMap::default(),
|
||||
@@ -355,22 +358,18 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::path_p2pk()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2pkh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2sh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2wpkh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2wsh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2tr())
|
||||
fs::create_dir_all(self.path_p2pk()).unwrap();
|
||||
fs::create_dir_all(self.path_p2pkh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2sh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2wpkh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2wsh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2tr())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset()
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"address_to_address_index"
|
||||
}
|
||||
|
||||
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
|
||||
mem::take(&mut self.p2pk)
|
||||
.into_values()
|
||||
@@ -424,7 +423,11 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
|
||||
self.open_all_p2tr();
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,8 @@ use std::{
|
||||
};
|
||||
|
||||
use crate::{
|
||||
io::Serialization,
|
||||
structs::{Counter, Date, Height},
|
||||
Serialization,
|
||||
};
|
||||
|
||||
#[derive(Default, Debug, Encode, Decode, Allocative)]
|
||||
@@ -35,10 +35,10 @@ impl DerefMut for Metadata {
|
||||
}
|
||||
|
||||
impl Metadata {
|
||||
pub fn import(path: PathBuf, version: u16) -> Self {
|
||||
pub fn import(path: &Path, version: u16) -> Self {
|
||||
Self {
|
||||
data: MetadataData::import(&path, version),
|
||||
path,
|
||||
data: MetadataData::import(path, version),
|
||||
path: path.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use std::thread::{self};
|
||||
|
||||
use allocative::Allocative;
|
||||
|
||||
mod _database;
|
||||
@@ -18,18 +16,14 @@ pub use address_index_to_address_data::*;
|
||||
pub use address_index_to_empty_address_data::*;
|
||||
pub use address_to_address_index::*;
|
||||
use itertools::Itertools;
|
||||
use log::info;
|
||||
use metadata::*;
|
||||
use rayon::iter::{IntoParallelIterator, ParallelIterator};
|
||||
pub use txid_to_tx_data::*;
|
||||
pub use txout_index_to_address_index::*;
|
||||
pub use txout_index_to_amount::*;
|
||||
|
||||
use crate::{
|
||||
log,
|
||||
structs::{Date, Height},
|
||||
utils::time,
|
||||
Exit,
|
||||
};
|
||||
use crate::structs::{Config, Date, Height};
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct Databases {
|
||||
@@ -42,18 +36,20 @@ pub struct Databases {
|
||||
}
|
||||
|
||||
impl Databases {
|
||||
pub fn import() -> Self {
|
||||
let address_index_to_address_data = AddressIndexToAddressData::init();
|
||||
pub fn import(config: &Config) -> Self {
|
||||
let address_index_to_address_data = AddressIndexToAddressData::init(config);
|
||||
|
||||
let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init();
|
||||
let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init(config);
|
||||
|
||||
let address_to_address_index = AddressToAddressIndex::init();
|
||||
let address_to_address_index = AddressToAddressIndex::init(config);
|
||||
|
||||
let txid_to_tx_data = TxidToTxData::init();
|
||||
let txid_to_tx_data = TxidToTxData::init(config);
|
||||
|
||||
let txout_index_to_address_index = TxoutIndexToAddressIndex::init();
|
||||
let txout_index_to_address_index = TxoutIndexToAddressIndex::init(config);
|
||||
|
||||
let txout_index_to_amount = TxoutIndexToAmount::init();
|
||||
let txout_index_to_amount = TxoutIndexToAmount::init(config);
|
||||
|
||||
info!("Imported databases");
|
||||
|
||||
Self {
|
||||
address_index_to_address_data,
|
||||
@@ -91,62 +87,21 @@ impl Databases {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
pub fn export(
|
||||
&mut self,
|
||||
height: Height,
|
||||
date: Date,
|
||||
defragment: bool,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.export_metadata(height, date)?;
|
||||
|
||||
self.drain_to_vec()
|
||||
.into_par_iter()
|
||||
.try_for_each(AnyDatabase::boxed_export)?;
|
||||
.try_for_each(|s| AnyDatabase::boxed_export(s, defragment))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
thread::scope(|s| {
|
||||
s.spawn(|| {
|
||||
self.address_index_to_address_data.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.address_index_to_empty_address_data.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.address_to_address_index.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.txid_to_tx_data.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.txout_index_to_address_index.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.txout_index_to_amount.open_all();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
pub fn defragment(&mut self, exit: &Exit) {
|
||||
exit.block();
|
||||
|
||||
log("Databases defragmentation");
|
||||
|
||||
time("Defragmenting databases", || {
|
||||
time("Opened all databases", || self.open_all());
|
||||
|
||||
log("Defragmenting...");
|
||||
|
||||
self.drain_to_vec()
|
||||
.into_par_iter()
|
||||
.for_each(AnyDatabase::boxed_defragment);
|
||||
});
|
||||
|
||||
exit.unblock();
|
||||
}
|
||||
|
||||
pub fn reset(&mut self, include_addresses: bool) {
|
||||
if include_addresses {
|
||||
let _ = self.address_index_to_address_data.reset();
|
||||
@@ -2,13 +2,14 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use biter::bitcoin::Txid;
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::structs::{Date, Height, TxData, U8x31};
|
||||
use crate::structs::{Config, TxData, U8x31};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
|
||||
@@ -18,8 +19,8 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct TxidToTxData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
map: BTreeMap<u16, Database>,
|
||||
}
|
||||
|
||||
@@ -109,8 +110,9 @@ impl TxidToTxData {
|
||||
|
||||
#[inline(always)]
|
||||
pub fn _open_db(&mut self, db_index: u16) -> &mut Database {
|
||||
let path = self.path.to_owned();
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let path = Self::root().join(db_index.to_string());
|
||||
let path = path.join(db_index.to_string());
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
}
|
||||
@@ -125,31 +127,23 @@ impl TxidToTxData {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for TxidToTxData {
|
||||
fn import() -> Self {
|
||||
let metadata = Metadata::import(Self::root(), 2);
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("txid_to_tx_data");
|
||||
let metadata = Metadata::import(&path, 2);
|
||||
|
||||
Self {
|
||||
path,
|
||||
metadata,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"txid_to_tx_data"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -180,7 +174,11 @@ impl AnyDatabaseGroup for TxidToTxData {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+16
-19
@@ -2,12 +2,13 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::structs::{Date, Height, TxoutIndex};
|
||||
use crate::structs::{Config, TxoutIndex};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
|
||||
@@ -17,8 +18,8 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct TxoutIndexToAddressIndex {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -69,6 +70,7 @@ impl TxoutIndexToAddressIndex {
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path.to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -77,7 +79,7 @@ impl TxoutIndexToAddressIndex {
|
||||
(db_index + 1) * DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
@@ -89,30 +91,21 @@ impl TxoutIndexToAddressIndex {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("txout_index_to_address_index");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"txout_index_to_address_index"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -151,7 +144,11 @@ impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+16
-19
@@ -2,12 +2,13 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::structs::{Amount, Date, Height, TxoutIndex};
|
||||
use crate::structs::{Amount, Config, TxoutIndex};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
|
||||
@@ -17,8 +18,8 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct TxoutIndexToAmount {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -69,6 +70,7 @@ impl TxoutIndexToAmount {
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path.to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -77,7 +79,7 @@ impl TxoutIndexToAmount {
|
||||
(db_index + 1) * DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
@@ -89,30 +91,21 @@ impl TxoutIndexToAmount {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for TxoutIndexToAmount {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("txout_index_to_amount");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"txout_index_to_amount"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -151,7 +144,11 @@ impl AnyDatabaseGroup for TxoutIndexToAmount {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -3,14 +3,14 @@ use rayon::prelude::*;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{
|
||||
parser::datasets::{
|
||||
cohort_metadata::AddressCohortMetadataDataset, ComputeData, DateRecapDataset, RatioDataset,
|
||||
SubDataset,
|
||||
},
|
||||
structs::{
|
||||
AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, BiMap, Date, Height, MapKind, Timestamp, OHLC,
|
||||
AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, BiMap, Date, DateMap, Height, HeightMap,
|
||||
MapKind, Timestamp, OHLC,
|
||||
},
|
||||
DateMap, HeightMap,
|
||||
};
|
||||
|
||||
use super::{AnyDatasetGroup, MinInitialStates};
|
||||
+4
-4
@@ -2,8 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -15,8 +15,8 @@ pub struct AllAddressesMetadataDataset {
|
||||
}
|
||||
|
||||
impl AllAddressesMetadataDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(path: &MapPath, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| path.join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -3,9 +3,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::{AddressCohortId, DurableStates},
|
||||
structs::{AddressSplit, BiMap, Config, Date, Height},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::{AddressCohortId, DurableStates},
|
||||
},
|
||||
structs::{AddressSplit, BiMap, Config, Date, Height, MapPath},
|
||||
};
|
||||
|
||||
use super::cohort_metadata::AddressCohortMetadataDataset;
|
||||
@@ -23,7 +25,7 @@ pub struct CohortDataset {
|
||||
|
||||
impl CohortDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
id: AddressCohortId,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -33,8 +35,8 @@ impl CohortDataset {
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
split,
|
||||
metadata: AddressCohortMetadataDataset::import(parent_path, &name, config)?,
|
||||
subs: SubDataset::import(parent_path, &name, config)?,
|
||||
metadata: AddressCohortMetadataDataset::import(path, &name, config)?,
|
||||
subs: SubDataset::import(path, &name, config)?,
|
||||
};
|
||||
|
||||
s.min_initial_states
|
||||
+5
-5
@@ -2,8 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -19,15 +19,15 @@ pub struct AddressCohortMetadataDataset {
|
||||
|
||||
impl AddressCohortMetadataDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -7,9 +7,8 @@ use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
states::SplitByAddressCohort,
|
||||
structs::{BiMap, Config, Height},
|
||||
Date,
|
||||
parser::states::SplitByAddressCohort,
|
||||
structs::{BiMap, Config, Date, Height},
|
||||
};
|
||||
|
||||
use self::{all_metadata::AllAddressesMetadataDataset, cohort::CohortDataset};
|
||||
@@ -26,13 +25,15 @@ pub struct AddressDatasets {
|
||||
}
|
||||
|
||||
impl AddressDatasets {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let mut cohorts = SplitByAddressCohort::<Option<CohortDataset>>::default();
|
||||
|
||||
let path_dataset = config.path_datasets();
|
||||
|
||||
cohorts
|
||||
.as_vec()
|
||||
.into_par_iter()
|
||||
.map(|(_, id)| (id, CohortDataset::import(parent_path, id, config)))
|
||||
.map(|(_, id)| (id, CohortDataset::import(&path_dataset, id, config)))
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
|
||||
@@ -43,7 +44,7 @@ impl AddressDatasets {
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
|
||||
metadata: AllAddressesMetadataDataset::import(parent_path, config)?,
|
||||
metadata: AllAddressesMetadataDataset::import(&path_dataset, config)?,
|
||||
|
||||
cohorts: cohorts.unwrap(),
|
||||
};
|
||||
@@ -2,7 +2,7 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, Date, HeightMap, MapKind, Timestamp},
|
||||
};
|
||||
|
||||
@@ -16,8 +16,8 @@ pub struct BlockMetadataDataset {
|
||||
}
|
||||
|
||||
impl BlockMetadataDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,9 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
structs::{Config, MapKind},
|
||||
DateMap, HeightMap,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, DateMap, HeightMap, MapKind},
|
||||
};
|
||||
|
||||
use super::{InsertData, MinInitialStates};
|
||||
@@ -17,8 +16,8 @@ pub struct CoindaysDataset {
|
||||
}
|
||||
|
||||
impl CoindaysDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,9 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
structs::{BiMap, Config, DateMap, Height, MapKind},
|
||||
structs::{BiMap, Config, DateMap, Height, HeightMap, MapKind},
|
||||
utils::{ONE_DAY_IN_DAYS, ONE_YEAR_IN_DAYS, THREE_MONTHS_IN_DAYS, TWO_WEEK_IN_DAYS},
|
||||
HeightMap,
|
||||
};
|
||||
|
||||
use super::{AnyDataset, ComputeData, InsertData, MinInitialStates, RatioDataset};
|
||||
@@ -72,8 +71,9 @@ pub struct CointimeDataset {
|
||||
}
|
||||
|
||||
impl CointimeDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let path_dataset = config.path_datasets();
|
||||
let f = |s: &str| path_dataset.join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -93,7 +93,7 @@ impl CointimeDataset {
|
||||
// Computed
|
||||
active_cap: BiMap::new_bin(1, MapKind::Computed, &f("active_cap")),
|
||||
active_price: BiMap::new_bin(1, MapKind::Computed, &f("active_price")),
|
||||
active_price_ratio: RatioDataset::import(parent_path, "active_price", config)?,
|
||||
active_price_ratio: RatioDataset::import(&path_dataset, "active_price", config)?,
|
||||
active_supply: BiMap::new_bin(1, MapKind::Computed, &f("active_supply")),
|
||||
active_supply_3m_net_change: BiMap::new_bin(
|
||||
1,
|
||||
@@ -140,7 +140,7 @@ impl CointimeDataset {
|
||||
),
|
||||
cointime_cap: BiMap::new_bin(1, MapKind::Computed, &f("cointime_cap")),
|
||||
cointime_price: BiMap::new_bin(1, MapKind::Computed, &f("cointime_price")),
|
||||
cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price", config)?,
|
||||
cointime_price_ratio: RatioDataset::import(&path_dataset, "cointime_price", config)?,
|
||||
cointime_value_created: HeightMap::new_bin(
|
||||
1,
|
||||
MapKind::Computed,
|
||||
@@ -237,7 +237,11 @@ impl CointimeDataset {
|
||||
&f("true_market_deviation"),
|
||||
),
|
||||
true_market_mean: BiMap::new_bin(1, MapKind::Computed, &f("true_market_mean")),
|
||||
true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean", config)?,
|
||||
true_market_mean_ratio: RatioDataset::import(
|
||||
&path_dataset,
|
||||
"true_market_mean",
|
||||
config,
|
||||
)?,
|
||||
true_market_net_unrealized_profit_and_loss: BiMap::new_bin(
|
||||
1,
|
||||
MapKind::Computed,
|
||||
@@ -245,7 +249,7 @@ impl CointimeDataset {
|
||||
),
|
||||
vaulted_cap: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_cap")),
|
||||
vaulted_price: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_price")),
|
||||
vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price", config)?,
|
||||
vaulted_price_ratio: RatioDataset::import(&path_dataset, "vaulted_price", config)?,
|
||||
vaulted_supply: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_supply")),
|
||||
vaulted_supply_3m_net_change: BiMap::new_bin(
|
||||
1,
|
||||
@@ -16,8 +16,8 @@ pub struct ConstantDataset {
|
||||
}
|
||||
|
||||
impl ConstantDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,7 +2,7 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, DateMap, Height, MapKind},
|
||||
};
|
||||
|
||||
@@ -17,8 +17,8 @@ pub struct DateMetadataDataset {
|
||||
}
|
||||
|
||||
impl DateMetadataDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -4,7 +4,7 @@ use ordered_float::OrderedFloat;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Amount, BiMap, Config, DateMap, Height, HeightMap, MapKey, MapKind},
|
||||
utils::{
|
||||
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
|
||||
@@ -123,8 +123,8 @@ pub struct MiningDataset {
|
||||
}
|
||||
|
||||
impl MiningDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -1,9 +1,10 @@
|
||||
use std::{collections::BTreeMap, fs, ops::RangeInclusive, path::Path};
|
||||
use std::{collections::BTreeMap, ops::RangeInclusive, path::PathBuf};
|
||||
|
||||
use allocative::Allocative;
|
||||
|
||||
use itertools::Itertools;
|
||||
|
||||
use log::info;
|
||||
use rayon::prelude::*;
|
||||
|
||||
mod _traits;
|
||||
@@ -34,16 +35,18 @@ pub use transaction::*;
|
||||
pub use utxo::*;
|
||||
|
||||
use crate::{
|
||||
databases::Databases,
|
||||
io::{Json, JSON_EXTENSION},
|
||||
states::{
|
||||
AddressCohortsInputStates,
|
||||
AddressCohortsOneShotStates,
|
||||
AddressCohortsRealizedStates,
|
||||
States,
|
||||
UTXOCohortsOneShotStates,
|
||||
// UTXOCohortsReceivedStates,
|
||||
UTXOCohortsSentStates,
|
||||
io::Json,
|
||||
parser::{
|
||||
databases::Databases,
|
||||
states::{
|
||||
AddressCohortsInputStates,
|
||||
AddressCohortsOneShotStates,
|
||||
AddressCohortsRealizedStates,
|
||||
States,
|
||||
UTXOCohortsOneShotStates,
|
||||
// UTXOCohortsReceivedStates,
|
||||
UTXOCohortsSentStates,
|
||||
},
|
||||
},
|
||||
structs::{Amount, Config, Date, Height, Price, Timestamp},
|
||||
};
|
||||
@@ -84,7 +87,7 @@ pub struct ComputeData<'a> {
|
||||
}
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AllDatasets {
|
||||
pub struct Datasets {
|
||||
min_initial_states: MinInitialStates,
|
||||
|
||||
pub constant: ConstantDataset,
|
||||
@@ -99,31 +102,27 @@ pub struct AllDatasets {
|
||||
pub utxo: UTXODatasets,
|
||||
}
|
||||
|
||||
const DATASETS_PATH: &str = "../datasets";
|
||||
|
||||
impl AllDatasets {
|
||||
impl Datasets {
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let path = DATASETS_PATH;
|
||||
let price = PriceDatasets::import(config)?;
|
||||
|
||||
let price = PriceDatasets::import(path, config)?;
|
||||
let constant = ConstantDataset::import(config)?;
|
||||
|
||||
let constant = ConstantDataset::import(path, config)?;
|
||||
let date_metadata = DateMetadataDataset::import(config)?;
|
||||
|
||||
let date_metadata = DateMetadataDataset::import(path, config)?;
|
||||
let cointime = CointimeDataset::import(config)?;
|
||||
|
||||
let cointime = CointimeDataset::import(path, config)?;
|
||||
let coindays = CoindaysDataset::import(config)?;
|
||||
|
||||
let coindays = CoindaysDataset::import(path, config)?;
|
||||
let mining = MiningDataset::import(config)?;
|
||||
|
||||
let mining = MiningDataset::import(path, config)?;
|
||||
let block_metadata = BlockMetadataDataset::import(config)?;
|
||||
|
||||
let block_metadata = BlockMetadataDataset::import(path, config)?;
|
||||
let transaction = TransactionDataset::import(config)?;
|
||||
|
||||
let transaction = TransactionDataset::import(path, config)?;
|
||||
let address = AddressDatasets::import(config)?;
|
||||
|
||||
let address = AddressDatasets::import(path, config)?;
|
||||
|
||||
let utxo = UTXODatasets::import(path, config)?;
|
||||
let utxo = UTXODatasets::import(config)?;
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -143,7 +142,7 @@ impl AllDatasets {
|
||||
s.min_initial_states
|
||||
.consume(MinInitialStates::compute_from_datasets(&s, config));
|
||||
|
||||
s.export_meta_files()?;
|
||||
info!("Imported datasets");
|
||||
|
||||
Ok(s)
|
||||
}
|
||||
@@ -269,43 +268,23 @@ impl AllDatasets {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn export_meta_files(&self) -> color_eyre::Result<()> {
|
||||
let mut path_to_type: BTreeMap<&Path, &str> = self
|
||||
pub fn get_paths_to_type(&self, config: &Config) -> BTreeMap<PathBuf, String> {
|
||||
let mut path_to_type: BTreeMap<PathBuf, String> = self
|
||||
.to_any_dataset_vec()
|
||||
.into_iter()
|
||||
.flat_map(|dataset| dataset.to_all_map_vec())
|
||||
.flat_map(|map| map.exported_path_with_t_name())
|
||||
.flat_map(|map| map.get_paths_to_type())
|
||||
.collect();
|
||||
|
||||
path_to_type.insert(Path::new("../datasets/last"), "Value");
|
||||
path_to_type.insert(
|
||||
config.path_datasets_last_values().unwrap().to_owned(),
|
||||
"Value".to_string(),
|
||||
);
|
||||
|
||||
let datasets_len = path_to_type.len();
|
||||
|
||||
let server_inputs_path = "../server/in";
|
||||
|
||||
fs::create_dir_all(server_inputs_path)?;
|
||||
|
||||
Json::export(
|
||||
Path::new(&format!("{server_inputs_path}/disk_path_to_type.json")),
|
||||
&path_to_type,
|
||||
)?;
|
||||
|
||||
let datasets_len_path = format!("{server_inputs_path}/datasets_len.txt");
|
||||
|
||||
if let Ok(len) = fs::read_to_string(&datasets_len_path) {
|
||||
if let Ok(len) = len.parse::<usize>() {
|
||||
if datasets_len == len {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs::write(datasets_len_path, datasets_len.to_string())?;
|
||||
|
||||
Ok(())
|
||||
path_to_type
|
||||
}
|
||||
|
||||
pub fn export(&mut self) -> color_eyre::Result<()> {
|
||||
pub fn export(&mut self, config: &Config) -> color_eyre::Result<()> {
|
||||
self.to_mut_any_dataset_vec()
|
||||
.into_iter()
|
||||
.for_each(|dataset| dataset.pre_export());
|
||||
@@ -316,6 +295,11 @@ impl AllDatasets {
|
||||
|
||||
let mut path_to_last: BTreeMap<String, Value> = BTreeMap::default();
|
||||
|
||||
let path_dataset = config.path_datasets();
|
||||
let path_dataset_ser = path_dataset.to_str().unwrap();
|
||||
let path_price = config.path_price();
|
||||
let path_price_ser = path_price.to_str().unwrap();
|
||||
|
||||
self.to_mut_any_dataset_vec()
|
||||
.into_iter()
|
||||
.for_each(|dataset| {
|
||||
@@ -326,34 +310,28 @@ impl AllDatasets {
|
||||
if let Some(last_value) = map.last_value() {
|
||||
let mut last_path = last_path.clone();
|
||||
last_path.pop();
|
||||
let key = last_path
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.replace(path_dataset_ser, "")
|
||||
.replace(path_price_ser, "")
|
||||
.split("/")
|
||||
.join("-")
|
||||
.to_string();
|
||||
|
||||
let last_path = last_path.to_str().unwrap();
|
||||
|
||||
let skip = if last_path.starts_with(DATASETS_PATH) {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
|
||||
path_to_last.insert(
|
||||
last_path.split('/').skip(skip).join("-").to_string(),
|
||||
last_value,
|
||||
);
|
||||
path_to_last.insert(key, last_value);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Json::export(
|
||||
Path::new(&format!("{DATASETS_PATH}/last.{JSON_EXTENSION}")),
|
||||
&path_to_last,
|
||||
)?;
|
||||
Json::export(&config.path_datasets_last_values(), &path_to_last)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl AnyDatasets for AllDatasets {
|
||||
impl AnyDatasets for Datasets {
|
||||
fn get_min_initial_states(&self) -> &MinInitialStates {
|
||||
&self.min_initial_states
|
||||
}
|
||||
@@ -7,7 +7,7 @@ use color_eyre::eyre::Error;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
price::{Binance, Kibo, Kraken},
|
||||
parser::price::{Binance, Kibo, Kraken},
|
||||
structs::{
|
||||
Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey,
|
||||
MapKind, Timestamp, OHLC,
|
||||
@@ -85,10 +85,9 @@ pub struct PriceDatasets {
|
||||
}
|
||||
|
||||
impl PriceDatasets {
|
||||
pub fn import(datasets_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let price_path = "../price";
|
||||
|
||||
let f = |s: &str| format!("{datasets_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let path_dataset = config.path_datasets();
|
||||
let f = |s: &str| path_dataset.join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -104,7 +103,7 @@ impl PriceDatasets {
|
||||
// ---
|
||||
// Inserted
|
||||
// ---
|
||||
ohlc: BiMap::new_json(1, MapKind::Inserted, price_path),
|
||||
ohlc: BiMap::new_json(1, MapKind::Inserted, &config.path_price()),
|
||||
|
||||
// ---
|
||||
// Computed
|
||||
@@ -115,31 +114,31 @@ impl PriceDatasets {
|
||||
close: BiMap::new_bin(1, MapKind::Computed, &f("close")),
|
||||
market_cap: BiMap::new_bin(1, MapKind::Computed, &f("market_cap")),
|
||||
price_1w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1w_sma")),
|
||||
price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma", config)?,
|
||||
price_1w_sma_ratio: RatioDataset::import(&path_dataset, "price_1w_sma", config)?,
|
||||
price_1m_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1m_sma")),
|
||||
price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma", config)?,
|
||||
price_1m_sma_ratio: RatioDataset::import(&path_dataset, "price_1m_sma", config)?,
|
||||
price_1y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1y_sma")),
|
||||
price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma", config)?,
|
||||
price_1y_sma_ratio: RatioDataset::import(&path_dataset, "price_1y_sma", config)?,
|
||||
price_2y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_2y_sma")),
|
||||
price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma", config)?,
|
||||
price_2y_sma_ratio: RatioDataset::import(&path_dataset, "price_2y_sma", config)?,
|
||||
price_4y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_4y_sma")),
|
||||
price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma", config)?,
|
||||
price_4y_sma_ratio: RatioDataset::import(&path_dataset, "price_4y_sma", config)?,
|
||||
price_8d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_8d_sma")),
|
||||
price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma", config)?,
|
||||
price_8d_sma_ratio: RatioDataset::import(&path_dataset, "price_8d_sma", config)?,
|
||||
price_13d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_13d_sma")),
|
||||
price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma", config)?,
|
||||
price_13d_sma_ratio: RatioDataset::import(&path_dataset, "price_13d_sma", config)?,
|
||||
price_21d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_21d_sma")),
|
||||
price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma", config)?,
|
||||
price_21d_sma_ratio: RatioDataset::import(&path_dataset, "price_21d_sma", config)?,
|
||||
price_34d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_34d_sma")),
|
||||
price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma", config)?,
|
||||
price_34d_sma_ratio: RatioDataset::import(&path_dataset, "price_34d_sma", config)?,
|
||||
price_55d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_55d_sma")),
|
||||
price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma", config)?,
|
||||
price_55d_sma_ratio: RatioDataset::import(&path_dataset, "price_55d_sma", config)?,
|
||||
price_89d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_89d_sma")),
|
||||
price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma", config)?,
|
||||
price_89d_sma_ratio: RatioDataset::import(&path_dataset, "price_89d_sma", config)?,
|
||||
price_144d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_144d_sma")),
|
||||
price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma", config)?,
|
||||
price_144d_sma_ratio: RatioDataset::import(&path_dataset, "price_144d_sma", config)?,
|
||||
price_200w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_200w_sma")),
|
||||
price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma", config)?,
|
||||
price_200w_sma_ratio: RatioDataset::import(&path_dataset, "price_200w_sma", config)?,
|
||||
price_1d_total_return: DateMap::new_bin(
|
||||
1,
|
||||
MapKind::Computed,
|
||||
@@ -540,6 +539,7 @@ impl PriceDatasets {
|
||||
height: Height,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if let Some(ohlc) = self.ohlc.height.get_or_import(&height) {
|
||||
return Ok(ohlc);
|
||||
@@ -558,7 +558,7 @@ impl PriceDatasets {
|
||||
.unwrap_or_else(|_| {
|
||||
self.get_from_1mn_binance(timestamp, previous_timestamp)
|
||||
.unwrap_or_else(|_| {
|
||||
self.get_from_har_binance(timestamp, previous_timestamp)
|
||||
self.get_from_har_binance(timestamp, previous_timestamp, config)
|
||||
.unwrap_or_else(|_| {
|
||||
self.get_from_height_kibo(&height).unwrap_or_else(|_| {
|
||||
let date = timestamp.to_date();
|
||||
@@ -659,10 +659,11 @@ How to fix this:
|
||||
&mut self,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if self.binance_har.is_none() {
|
||||
self.binance_har
|
||||
.replace(Binance::read_har_file().unwrap_or_default());
|
||||
.replace(Binance::read_har_file(config).unwrap_or_default());
|
||||
}
|
||||
|
||||
Self::find_height_ohlc(
|
||||
+9
-7
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::CapitalizationState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::CapitalizationState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
utils::ONE_MONTH_IN_DAYS,
|
||||
};
|
||||
|
||||
@@ -22,15 +24,15 @@ pub struct CapitalizationDataset {
|
||||
|
||||
impl CapitalizationDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -52,7 +54,7 @@ impl CapitalizationDataset {
|
||||
),
|
||||
realized_price: BiMap::new_bin(1, MapKind::Computed, &f("realized_price")),
|
||||
realized_price_ratio: RatioDataset::import(
|
||||
parent_path,
|
||||
path,
|
||||
&format!(
|
||||
"{}realized_price",
|
||||
name.as_ref().map_or("".to_owned(), |n| format!("{n}-"))
|
||||
@@ -2,10 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::InputState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
DateMap, HeightMap,
|
||||
parser::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::InputState,
|
||||
},
|
||||
structs::{BiMap, Config, DateMap, HeightMap, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -22,15 +23,15 @@ pub struct InputSubDataset {
|
||||
|
||||
impl InputSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -21,7 +21,10 @@ pub use supply::*;
|
||||
pub use unrealized::*;
|
||||
pub use utxo::*;
|
||||
|
||||
use crate::{datasets::AnyDataset, structs::Config};
|
||||
use crate::{
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, MapPath},
|
||||
};
|
||||
|
||||
use super::AnyDatasetGroup;
|
||||
|
||||
@@ -39,7 +42,7 @@ pub struct SubDataset {
|
||||
|
||||
impl SubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
parent_path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::PricePaidState,
|
||||
structs::{BiMap, Config, Date, Height, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::PricePaidState,
|
||||
},
|
||||
structs::{BiMap, Config, Date, Height, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -34,15 +36,15 @@ pub struct PricePaidSubDataset {
|
||||
|
||||
impl PricePaidSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,8 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::datasets::{AnyDataset, ComputeData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
|
||||
};
|
||||
|
||||
@@ -31,9 +31,9 @@ pub struct RatioDataset {
|
||||
}
|
||||
|
||||
impl RatioDataset {
|
||||
pub fn import(parent_path: &str, name: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f_ratio = |s: &str| format!("{parent_path}/market_price_to_{name}_{s}");
|
||||
let f_price = |s: &str| format!("{parent_path}/{name}_{s}");
|
||||
pub fn import(path: &MapPath, name: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f_ratio = |s: &str| path.join(&format!("market_price_to_{name}_{s}"));
|
||||
let f_price = |s: &str| path.join(&format!("{name}_{s}"));
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,11 +2,12 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::RealizedState,
|
||||
structs::{BiMap, Config, MapKind, Price},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::RealizedState,
|
||||
},
|
||||
structs::{BiMap, Config, DateMap, HeightMap, MapKind, MapPath, Price},
|
||||
utils::ONE_MONTH_IN_DAYS,
|
||||
DateMap, HeightMap,
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -45,18 +46,17 @@ pub struct RealizedSubDataset {
|
||||
|
||||
impl RealizedSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
|
||||
@@ -3,9 +3,11 @@ use std::{iter::Sum, ops::Add};
|
||||
use allocative::Allocative;
|
||||
|
||||
use crate::{
|
||||
structs::{DateMapChunkId, GenericMap, MapKey, MapKind, MapSerialized, MapValue},
|
||||
structs::{
|
||||
Date, DateMapChunkId, GenericMap, MapChunkId, MapKey, MapKind, MapPath, MapSerialized,
|
||||
MapValue, SerializedBTreeMap,
|
||||
},
|
||||
utils::{get_percentile, LossyFrom},
|
||||
Date, MapChunkId, SerializedBTreeMap,
|
||||
};
|
||||
|
||||
pub type DateRecapDataset<T> = RecapDataset<Date, T, DateMapChunkId, SerializedBTreeMap<Date, T>>;
|
||||
@@ -91,8 +93,8 @@ where
|
||||
Key: MapKey<ChunkId>,
|
||||
Serialized: MapSerialized<Key, Value, ChunkId>,
|
||||
{
|
||||
pub fn import(parent_path: &str, options: RecapOptions) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(path: &MapPath, options: RecapOptions) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| path.join(s);
|
||||
|
||||
let s = Self {
|
||||
// ---
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::SupplyState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::SupplyState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -19,15 +21,15 @@ pub struct SupplySubDataset {
|
||||
|
||||
impl SupplySubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::UnrealizedState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::UnrealizedState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -26,15 +28,15 @@ pub struct UnrealizedSubDataset {
|
||||
|
||||
impl UnrealizedSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::UTXOState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::UTXOState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -16,15 +18,15 @@ pub struct UTXOSubDataset {
|
||||
|
||||
impl UTXOSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,12 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::InsertData,
|
||||
structs::{BiMap, Config, HeightMap, MapKind},
|
||||
parser::datasets::InsertData,
|
||||
structs::{BiMap, Config, DateMap, HeightMap, MapKind},
|
||||
utils::{
|
||||
ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, TARGET_BLOCKS_PER_DAY,
|
||||
},
|
||||
DateMap,
|
||||
};
|
||||
|
||||
use super::{AnyDataset, ComputeData, MinInitialStates};
|
||||
@@ -52,8 +51,8 @@ pub struct TransactionDataset {
|
||||
}
|
||||
|
||||
impl TransactionDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::UTXOCohortId,
|
||||
structs::{BiMap, Config, Date, Height},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::UTXOCohortId,
|
||||
},
|
||||
structs::{BiMap, Config, Date, Height, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -18,7 +20,7 @@ pub struct UTXODataset {
|
||||
|
||||
impl UTXODataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
parent_path: &MapPath,
|
||||
id: UTXOCohortId,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -7,8 +7,8 @@ use rayon::prelude::*;
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDatasets,
|
||||
states::{SplitByUTXOCohort, UTXOCohortId},
|
||||
parser::datasets::AnyDatasets,
|
||||
parser::states::{SplitByUTXOCohort, UTXOCohortId},
|
||||
structs::{BiMap, Config, Date, Height},
|
||||
};
|
||||
|
||||
@@ -22,13 +22,15 @@ pub struct UTXODatasets {
|
||||
}
|
||||
|
||||
impl UTXODatasets {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let mut cohorts = SplitByUTXOCohort::<Option<UTXODataset>>::default();
|
||||
|
||||
let path_dataset = config.path_datasets();
|
||||
|
||||
cohorts
|
||||
.as_vec()
|
||||
.into_par_iter()
|
||||
.map(|(_, id)| (id, UTXODataset::import(parent_path, id, config)))
|
||||
.map(|(_, id)| (id, UTXODataset::import(&path_dataset, id, config)))
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
|
||||
@@ -0,0 +1,52 @@
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use biter::bitcoincore_rpc::{Client, RpcApi};
|
||||
|
||||
mod actions;
|
||||
mod databases;
|
||||
mod datasets;
|
||||
mod price;
|
||||
mod states;
|
||||
|
||||
pub use actions::*;
|
||||
pub use databases::*;
|
||||
pub use datasets::*;
|
||||
use log::info;
|
||||
pub use states::*;
|
||||
|
||||
use crate::structs::{Config, Exit};
|
||||
|
||||
pub fn main(
|
||||
config: &Config,
|
||||
rpc: &Client,
|
||||
exit: &Exit,
|
||||
mut databases: Databases,
|
||||
mut datasets: Datasets,
|
||||
) -> color_eyre::Result<()> {
|
||||
loop {
|
||||
let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
|
||||
|
||||
info!("{block_count} blocks found.");
|
||||
|
||||
iter_blocks(
|
||||
config,
|
||||
rpc,
|
||||
block_count,
|
||||
exit.clone(),
|
||||
&mut databases,
|
||||
&mut datasets,
|
||||
)?;
|
||||
|
||||
if let Some(delay) = config.delay {
|
||||
sleep(Duration::from_secs(delay))
|
||||
}
|
||||
|
||||
info!("Waiting for a new block...");
|
||||
|
||||
while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
@@ -1,26 +1,29 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::{collections::BTreeMap, fs, path::Path};
|
||||
use std::{collections::BTreeMap, fs};
|
||||
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use itertools::Itertools;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
io::{Json, INPUTS_FOLDER_PATH},
|
||||
structs::{Date, Timestamp, OHLC},
|
||||
utils::{log, retry},
|
||||
io::Json,
|
||||
structs::{Config, Date, Timestamp, OHLC},
|
||||
utils::retry,
|
||||
};
|
||||
|
||||
pub struct Binance;
|
||||
|
||||
impl Binance {
|
||||
pub fn read_har_file() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
log("binance: read har file");
|
||||
pub fn read_har_file(config: &Config) -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
info!("binance: read har file");
|
||||
|
||||
fs::create_dir_all(INPUTS_FOLDER_PATH)?;
|
||||
let path = config.path_inputs();
|
||||
|
||||
let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har");
|
||||
fs::create_dir_all(&path)?;
|
||||
|
||||
let path_binance_har = path.join("binance.har");
|
||||
|
||||
let json: BTreeMap<String, Value> = Json::import(&path_binance_har).unwrap_or_default();
|
||||
|
||||
@@ -104,7 +107,7 @@ impl Binance {
|
||||
}
|
||||
|
||||
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
log("binance: fetch 1mn");
|
||||
info!("binance: fetch 1mn");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -151,7 +154,7 @@ impl Binance {
|
||||
}
|
||||
|
||||
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
log("binance: fetch 1d");
|
||||
info!("binance: fetch 1d");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -3,12 +3,12 @@ use std::{collections::BTreeMap, str::FromStr};
|
||||
use chrono::NaiveDate;
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use itertools::Itertools;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
structs::{Date, DateMapChunkId, HeightMapChunkId, OHLC},
|
||||
utils::{log, retry},
|
||||
MapChunkId,
|
||||
structs::{Date, DateMapChunkId, HeightMapChunkId, MapChunkId, OHLC},
|
||||
utils::retry,
|
||||
};
|
||||
|
||||
pub struct Kibo;
|
||||
@@ -28,7 +28,7 @@ impl Kibo {
|
||||
}
|
||||
|
||||
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
|
||||
log("kibo: fetch height prices");
|
||||
info!("kibo: fetch height prices");
|
||||
|
||||
retry(
|
||||
|try_index| {
|
||||
@@ -61,7 +61,7 @@ impl Kibo {
|
||||
}
|
||||
|
||||
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
log("kibo: fetch date prices");
|
||||
info!("kibo: fetch date prices");
|
||||
|
||||
retry(
|
||||
|try_index| {
|
||||
@@ -1,18 +1,19 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
structs::{Date, Timestamp, OHLC},
|
||||
utils::{log, retry},
|
||||
utils::retry,
|
||||
};
|
||||
|
||||
pub struct Kraken;
|
||||
|
||||
impl Kraken {
|
||||
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
log("kraken: fetch 1mn");
|
||||
info!("kraken: fetch 1mn");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -66,7 +67,7 @@ impl Kraken {
|
||||
}
|
||||
|
||||
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
log("fetch kraken daily");
|
||||
info!("fetch kraken daily");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -0,0 +1,39 @@
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
fs, io,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use bincode::{Decode, Encode};
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
|
||||
use crate::{io::Serialization, structs::Config};
|
||||
|
||||
// https://github.com/djkoloski/rust_serialization_benchmark
|
||||
pub trait AnyState
|
||||
where
|
||||
Self: Debug + Encode + Decode + Serialize + DeserializeOwned,
|
||||
{
|
||||
fn name<'a>() -> &'a str;
|
||||
|
||||
fn path(config: &Config) -> PathBuf {
|
||||
config.path_states().join(Self::name())
|
||||
}
|
||||
|
||||
fn reset(&mut self, config: &Config) -> color_eyre::Result<(), io::Error> {
|
||||
self.clear();
|
||||
fs::remove_file(Self::path(config))
|
||||
}
|
||||
|
||||
fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let path = Self::path(config);
|
||||
fs::create_dir_all(&path)?;
|
||||
Serialization::Binary.import(&path)
|
||||
}
|
||||
|
||||
fn export(&self, config: &Config) -> color_eyre::Result<()> {
|
||||
Serialization::Binary.export(Path::new(&Self::path(config)), self)
|
||||
}
|
||||
|
||||
fn clear(&mut self);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user