mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-12 07:53:32 -07:00
global: snapshot
This commit is contained in:
k
+9
-1
@@ -12,7 +12,11 @@
|
||||
## Datasets
|
||||
|
||||
- Added `Sell Side Risk Ratio` to all entities
|
||||
- Removed datasets split by liquidity for all datasets already split by any address kind, while fun to have it took time to compute and space to store and no one was checking them
|
||||
- Added `Adjusted Value Created`, `Adjusted Value Destroyed` and `Adjusted Spent Output Profit Ratio` to all entities
|
||||
- Added `Satoshis Per Dollar`
|
||||
- Added `All Time High`
|
||||
- Added `Drawdown`
|
||||
- Removed datasets split by liquidity for all datasets already split by any address kind, while fun to have, they took time to compute, ram, and space to store and no one was actually checking them
|
||||
- Fixed a lot of values in split by liquidity datasets
|
||||
|
||||
## Website
|
||||
@@ -37,6 +41,10 @@
|
||||
- Fixed broken last values routes
|
||||
- Add support for the `/datasets/last` file via the `/api/last` route
|
||||
|
||||
## Biter
|
||||
|
||||
- Moved back to this repo
|
||||
|
||||
## v. 0.4.0 | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
|
||||
|
||||
|
||||
Generated
+466
@@ -0,0 +1,466 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
|
||||
|
||||
[[package]]
|
||||
name = "base58ck"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
|
||||
dependencies = [
|
||||
"bitcoin-internals",
|
||||
"bitcoin_hashes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
|
||||
|
||||
[[package]]
|
||||
name = "bech32"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin"
|
||||
version = "0.32.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea507acc1cd80fc084ace38544bbcf7ced7c2aa65b653b102de0ce718df668f6"
|
||||
dependencies = [
|
||||
"base58ck",
|
||||
"bech32",
|
||||
"bitcoin-internals",
|
||||
"bitcoin-io",
|
||||
"bitcoin-units",
|
||||
"bitcoin_hashes",
|
||||
"hex-conservative",
|
||||
"hex_lit",
|
||||
"secp256k1",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin-internals"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin-io"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "340e09e8399c7bd8912f495af6aa58bea0c9214773417ffaa8f6460f93aaee56"
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin-units"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2"
|
||||
dependencies = [
|
||||
"bitcoin-internals",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin_hashes"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16"
|
||||
dependencies = [
|
||||
"bitcoin-io",
|
||||
"hex-conservative",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoincore-rpc"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee"
|
||||
dependencies = [
|
||||
"bitcoincore-rpc-json",
|
||||
"jsonrpc",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoincore-rpc-json"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "biter"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"bitcoincore-rpc",
|
||||
"crossbeam",
|
||||
"derived-deref",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
|
||||
|
||||
[[package]]
|
||||
name = "derived-deref"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex-conservative"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex_lit"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
|
||||
|
||||
[[package]]
|
||||
name = "jsonrpc"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"minreq",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.155"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
|
||||
[[package]]
|
||||
name = "minreq"
|
||||
version = "2.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0"
|
||||
dependencies = [
|
||||
"log",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.86"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
||||
|
||||
[[package]]
|
||||
name = "secp256k1"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e0cc0f1cf93f4969faf3ea1c7d8a9faed25918d96affa959720823dfe86d4f3"
|
||||
dependencies = [
|
||||
"bitcoin_hashes",
|
||||
"rand",
|
||||
"secp256k1-sys",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "secp256k1-sys"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1433bd67156263443f14d603720b082dd3121779323fce20cba2aa07b874bc1b"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.204"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.204"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.72"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "biter"
|
||||
description = "A very fast Bitcoin block iterator"
|
||||
version = "0.1.1"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/kibo-money/kibo/tree/main/biter"
|
||||
keywords = ["bitcoin", "block", "iterator"]
|
||||
categories = ["cryptography::cryptocurrencies", "encoding"]
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
bitcoin = { version = "0.32.2", features = ["serde"] }
|
||||
rayon = "1.10.0"
|
||||
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
|
||||
serde = { version = "1.0.204", features = ["derive"] }
|
||||
serde_json = "1.0.122"
|
||||
derived-deref = "2.1.0"
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
# tokio = { version = "1.39.2", features = ["rt-multi-thread"] }
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Biter
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,62 @@
|
||||
# Biter
|
||||
|
||||
Biter (Bitcoin Block Iterator) is a very fast and simple Rust library which reads raw block files (*blkXXXXX.dat*) from Bitcoin Core Node and creates an iterator over all the requested blocks in sequential order (0, 1, 2, ...).
|
||||
|
||||
The element returned by the iterator is a tuple which includes the:
|
||||
- Height: `usize`
|
||||
- Block: `Block` (from `bitcoin-rust`)
|
||||
- Block's Hash: `BlockHash` (also from `bitcoin-rust`)
|
||||
|
||||
## Example
|
||||
|
||||
```rust
|
||||
use bitcoincore_rpc::{Auth, Client};
|
||||
|
||||
fn main() {
|
||||
let i = std::time::Instant::now();
|
||||
|
||||
// Path to the Bitcoin data directory
|
||||
let data_dir = "../../bitcoin";
|
||||
|
||||
// Path to the export directory where a mini blk indexer will be exported
|
||||
let export_dir = "./target";
|
||||
|
||||
// Inclusive starting height of the blocks received, `None` for 0
|
||||
let start = Some(850_000);
|
||||
|
||||
// Inclusive ending height of the blocks received, `None` for the last one
|
||||
let end = None;
|
||||
|
||||
// RPC client to filter out forks
|
||||
let url = "http://localhost:8332";
|
||||
let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
let rpc = Client::new(url, auth).unwrap();
|
||||
|
||||
// Create channel receiver then iterate over the blocks
|
||||
biter::new(data_dir, export_dir, start, end, rpc)
|
||||
.iter()
|
||||
.for_each(|(height, _block, hash)| {
|
||||
println!("{height}: {hash}");
|
||||
});
|
||||
|
||||
dbg!(i.elapsed());
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
Even though it reads *blkXXXXX.dat* files, it **needs** `bitcoind` to run with the RPC server to filter out block forks.
|
||||
|
||||
Peak memory should be around 500MB.
|
||||
|
||||
## Comparaison
|
||||
|
||||
| | [biter](https://crates.io/crates/biter) | [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) | [blocks_iterator](https://crates.io/crates/blocks_iterator) |
|
||||
| --- | --- | --- | --- |
|
||||
| Run **with** `bitcoind` | Yes ✅ | No ❌ | Yes ✅ |
|
||||
| Run **without** `bitcoind` | No ❌ | Yes ✅ | Yes ✅ |
|
||||
| `0..=855_000` | 16mn40s | 17mn 46s | > 2h |
|
||||
| `800_000..=855_000` | 2mn 53s (16mn40s if first run) | 3mn 2s | > 2h |
|
||||
|
||||
*Benchmarked on a Macbook Pro M3 Pro*
|
||||
@@ -0,0 +1,105 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::BTreeMap,
|
||||
fs::{self, File},
|
||||
io::{BufReader, BufWriter},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use derived_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{blk_recap::BlkRecap, BlkMetadataAndBlock};
|
||||
|
||||
#[derive(Deref, DerefMut, Debug)]
|
||||
pub struct BlkIndexToBlkRecap {
|
||||
path: String,
|
||||
#[target]
|
||||
tree: BTreeMap<usize, BlkRecap>,
|
||||
}
|
||||
|
||||
impl BlkIndexToBlkRecap {
|
||||
pub fn import(blocks_dir: &BTreeMap<usize, PathBuf>, export_dir: &str) -> Self {
|
||||
let path = format!("{export_dir}/blk_index_to_blk_recap.json");
|
||||
|
||||
let tree = {
|
||||
fs::create_dir_all(export_dir).unwrap();
|
||||
|
||||
if let Ok(file) = File::open(&path) {
|
||||
let reader = BufReader::new(file);
|
||||
serde_json::from_reader(reader).unwrap_or_default()
|
||||
} else {
|
||||
BTreeMap::default()
|
||||
}
|
||||
};
|
||||
|
||||
let mut this = Self { path, tree };
|
||||
|
||||
this.clean_outdated(blocks_dir);
|
||||
|
||||
this
|
||||
}
|
||||
|
||||
pub fn clean_outdated(&mut self, blocks_dir: &BTreeMap<usize, PathBuf>) {
|
||||
blocks_dir.iter().for_each(|(blk_index, blk_path)| {
|
||||
if let Some(blk_recap) = self.get(blk_index) {
|
||||
if blk_recap.has_different_modified_time(blk_path) {
|
||||
self.remove(blk_index);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn get_start_recap(&self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
|
||||
if let Some(start) = start {
|
||||
let (last_key, last_value) = self.last_key_value()?;
|
||||
|
||||
if last_value.height() < start {
|
||||
return Some((*last_key, *last_value));
|
||||
} else if let Some((blk_index, _)) = self
|
||||
.iter()
|
||||
.find(|(_, blk_recap)| blk_recap.is_younger_than(start))
|
||||
{
|
||||
if *blk_index != 0 {
|
||||
let blk_index = *blk_index - 1;
|
||||
return Some((blk_index, *self.get(&blk_index).unwrap()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: usize) {
|
||||
let blk_index = blk_metadata_and_block.blk_metadata.index;
|
||||
|
||||
if let Some(last_entry) = self.last_entry() {
|
||||
// if last_entry.get().is_older_than(height) {
|
||||
match last_entry.key().cmp(&blk_index) {
|
||||
Ordering::Greater => {
|
||||
last_entry.remove_entry();
|
||||
}
|
||||
Ordering::Less => {
|
||||
self.insert(blk_index, BlkRecap::from(height, blk_metadata_and_block));
|
||||
}
|
||||
Ordering::Equal => {}
|
||||
};
|
||||
// }
|
||||
} else {
|
||||
if blk_index != 0 || height != 0 {
|
||||
// dbg!(blk_index, height);
|
||||
unreachable!();
|
||||
}
|
||||
|
||||
self.insert(blk_index, BlkRecap::first(blk_metadata_and_block));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn export(&self) {
|
||||
let file = File::create(&self.path).unwrap_or_else(|_| {
|
||||
dbg!(&self.path);
|
||||
panic!("No such file or directory")
|
||||
});
|
||||
|
||||
serde_json::to_writer_pretty(&mut BufWriter::new(file), &self.tree).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::path_to_modified_time;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct BlkMetadata {
|
||||
pub index: usize,
|
||||
pub modified_time: u64,
|
||||
}
|
||||
|
||||
impl BlkMetadata {
|
||||
pub fn new(index: usize, path: &PathBuf) -> Self {
|
||||
Self {
|
||||
index,
|
||||
modified_time: path_to_modified_time(path),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
use bitcoin::Block;
|
||||
|
||||
use crate::BlkMetadata;
|
||||
|
||||
pub struct BlkMetadataAndBlock {
|
||||
pub blk_metadata: BlkMetadata,
|
||||
pub block: Block,
|
||||
}
|
||||
|
||||
impl BlkMetadataAndBlock {
|
||||
pub fn new(blk_metadata: BlkMetadata, block: Block) -> Self {
|
||||
Self {
|
||||
blk_metadata,
|
||||
block,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use bitcoin::{hashes::Hash, BlockHash};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{path_to_modified_time, BlkMetadataAndBlock};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct BlkRecap {
|
||||
min_continuous_height: usize,
|
||||
min_continuous_prev_hash: BlockHash,
|
||||
modified_time: u64,
|
||||
}
|
||||
|
||||
impl BlkRecap {
|
||||
pub fn first(blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
|
||||
Self {
|
||||
min_continuous_height: 0,
|
||||
min_continuous_prev_hash: BlockHash::all_zeros(),
|
||||
modified_time: blk_metadata_and_block.blk_metadata.modified_time,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from(height: usize, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
|
||||
Self {
|
||||
min_continuous_height: height,
|
||||
min_continuous_prev_hash: blk_metadata_and_block.block.header.prev_blockhash,
|
||||
modified_time: blk_metadata_and_block.blk_metadata.modified_time,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn has_different_modified_time(&self, blk_path: &PathBuf) -> bool {
|
||||
self.modified_time != path_to_modified_time(blk_path)
|
||||
}
|
||||
|
||||
pub fn is_younger_than(&self, height: usize) -> bool {
|
||||
self.min_continuous_height > height
|
||||
}
|
||||
|
||||
pub fn height(&self) -> usize {
|
||||
self.min_continuous_height
|
||||
}
|
||||
|
||||
pub fn prev_hash(&self) -> &BlockHash {
|
||||
&self.min_continuous_prev_hash
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet, VecDeque},
|
||||
fs::{self},
|
||||
ops::ControlFlow,
|
||||
thread,
|
||||
};
|
||||
|
||||
use bitcoin::{
|
||||
consensus::{Decodable, ReadExt},
|
||||
hashes::Hash,
|
||||
io::{Cursor, Read},
|
||||
Block, BlockHash,
|
||||
};
|
||||
use bitcoincore_rpc::RpcApi;
|
||||
use crossbeam::channel::{bounded, Receiver};
|
||||
use rayon::prelude::*;
|
||||
|
||||
pub use bitcoin;
|
||||
pub use bitcoincore_rpc;
|
||||
|
||||
mod blk_index_to_blk_recap;
|
||||
mod blk_metadata;
|
||||
mod blk_metadata_and_block;
|
||||
mod blk_recap;
|
||||
mod utils;
|
||||
|
||||
use blk_index_to_blk_recap::*;
|
||||
use blk_metadata::*;
|
||||
use blk_metadata_and_block::*;
|
||||
use utils::*;
|
||||
|
||||
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
|
||||
const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
|
||||
const BOUND_CAP: usize = 210;
|
||||
|
||||
enum BlockState {
|
||||
Raw(Vec<u8>),
|
||||
Decoded(Block),
|
||||
}
|
||||
|
||||
///
|
||||
/// Returns a crossbeam channel receiver that receives `(usize, Block, BlockHash)` tuples (with `usize` being the height) in sequential order.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `data_dir` - Path to the Bitcoin data directory
|
||||
/// * `export_dir` - Path to the export directory where a mini blk indexer will be exported
|
||||
/// * `start` - Inclusive starting height of the blocks received, `None` for 0
|
||||
/// * `end` - Inclusive ending height of the blocks received, `None` for the last one
|
||||
/// * `rpc` - RPC client to filter out forks
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use bitcoincore_rpc::{Auth, Client};
|
||||
///
|
||||
/// fn main() {
|
||||
/// let i = std::time::Instant::now();
|
||||
///
|
||||
/// let url = "http://localhost:8332";
|
||||
/// let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
/// let rpc = Client::new(url, auth).unwrap();
|
||||
///
|
||||
/// let data_dir = "../../bitcoin";
|
||||
/// let export_dir = "./target";
|
||||
/// let start = Some(850_000);
|
||||
/// let end = None;
|
||||
///
|
||||
/// biter::new(data_dir, export_dir, start, end, rpc)
|
||||
/// .iter()
|
||||
/// .for_each(|(height, _block, hash)| {
|
||||
/// println!("{height}: {hash}");
|
||||
/// });
|
||||
///
|
||||
/// dbg!(i.elapsed());
|
||||
///}
|
||||
/// ```
|
||||
///
|
||||
pub fn new(
|
||||
data_dir: &str,
|
||||
export_dir: &str,
|
||||
start: Option<usize>,
|
||||
end: Option<usize>,
|
||||
rpc: bitcoincore_rpc::Client,
|
||||
) -> Receiver<(usize, Block, BlockHash)> {
|
||||
let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP);
|
||||
let (send_block, recv_block) = bounded(BOUND_CAP);
|
||||
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
|
||||
|
||||
let blocks_dir = scan_blocks_dir(data_dir);
|
||||
|
||||
let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blocks_dir, export_dir);
|
||||
|
||||
let start_recap = blk_index_to_blk_recap.get_start_recap(start);
|
||||
let starting_blk_index = start_recap.as_ref().map_or(0, |(index, _)| *index);
|
||||
|
||||
thread::spawn(move || {
|
||||
blocks_dir
|
||||
.into_iter()
|
||||
.filter(|(blk_index, _)| blk_index >= &starting_blk_index)
|
||||
.try_for_each(move |(blk_index, blk_path)| {
|
||||
let blk_metadata = BlkMetadata::new(blk_index, &blk_path);
|
||||
|
||||
let blk_bytes = fs::read(&blk_path).unwrap();
|
||||
let blk_bytes_len = blk_bytes.len() as u64;
|
||||
|
||||
let mut cursor = Cursor::new(blk_bytes.as_slice());
|
||||
|
||||
let mut current_4bytes = [0; 4];
|
||||
|
||||
'parent: loop {
|
||||
if cursor.position() == blk_bytes_len {
|
||||
break;
|
||||
}
|
||||
|
||||
// Read until we find a valid suite of MAGIC_BYTES
|
||||
loop {
|
||||
current_4bytes.rotate_left(1);
|
||||
|
||||
if let Ok(byte) = cursor.read_u8() {
|
||||
current_4bytes[3] = byte;
|
||||
} else {
|
||||
break 'parent;
|
||||
}
|
||||
|
||||
if current_4bytes == MAGIC_BYTES {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let block_size = cursor.read_u32().unwrap();
|
||||
|
||||
let mut raw_block = vec![0u8; block_size as usize];
|
||||
|
||||
cursor.read_exact(&mut raw_block).unwrap();
|
||||
|
||||
if send_block_reader
|
||||
.send((blk_metadata, BlockState::Raw(raw_block)))
|
||||
.is_err()
|
||||
{
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
})
|
||||
});
|
||||
|
||||
// thread::spawn(move || {
|
||||
// recv_block_reader.iter().par_bridge().try_for_each(
|
||||
// move |(blk_metadata, mut block_state)| {
|
||||
// let raw_block = match block_state {
|
||||
// BlockState::Raw(vec) => vec,
|
||||
// _ => unreachable!(),
|
||||
// };
|
||||
|
||||
// let mut cursor = Cursor::new(raw_block);
|
||||
|
||||
// block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
|
||||
|
||||
// if send_block
|
||||
// .send(BlkMetadataAndBlock::new(
|
||||
// blk_metadata,
|
||||
// match block_state {
|
||||
// BlockState::Decoded(block) => block,
|
||||
// _ => unreachable!(),
|
||||
// },
|
||||
// ))
|
||||
// .is_err()
|
||||
// {
|
||||
// return ControlFlow::Break(());
|
||||
// }
|
||||
|
||||
// ControlFlow::Continue(())
|
||||
// },
|
||||
// );
|
||||
// });
|
||||
|
||||
// Can't use the previous code because .send() blocks all the threads if full
|
||||
// And other .par_iter() are also stuck because of that
|
||||
thread::spawn(move || {
|
||||
let mut bulk = vec![];
|
||||
|
||||
let drain_and_send = |bulk: &mut Vec<_>| {
|
||||
// Using a vec and sending after to not end up with stuck threads in par iter
|
||||
bulk.par_iter_mut().for_each(|(_, block_state)| {
|
||||
let raw_block = match block_state {
|
||||
BlockState::Raw(vec) => vec,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let mut cursor = Cursor::new(raw_block);
|
||||
|
||||
*block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
|
||||
});
|
||||
|
||||
bulk.drain(..).try_for_each(|(blk_metadata, block_state)| {
|
||||
let block = match block_state {
|
||||
BlockState::Decoded(block) => block,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
if send_block
|
||||
.send(BlkMetadataAndBlock::new(blk_metadata, block))
|
||||
.is_err()
|
||||
{
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
})
|
||||
};
|
||||
|
||||
recv_block_reader.iter().try_for_each(|tuple| {
|
||||
bulk.push(tuple);
|
||||
|
||||
if bulk.len() < BOUND_CAP / 2 {
|
||||
return ControlFlow::Continue(());
|
||||
}
|
||||
|
||||
drain_and_send(&mut bulk)
|
||||
});
|
||||
|
||||
drain_and_send(&mut bulk)
|
||||
});
|
||||
|
||||
// Tokio version: 1022s
|
||||
// Slighlty slower than rayon version
|
||||
// thread::spawn(move || {
|
||||
// let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
// let _guard = rt.enter();
|
||||
|
||||
// let mut tasks = VecDeque::with_capacity(BOUND);
|
||||
|
||||
// recv_block_reader
|
||||
// .iter()
|
||||
// .try_for_each(move |(blk_metadata, block_state)| {
|
||||
// let raw_block = match block_state {
|
||||
// BlockState::Raw(vec) => vec,
|
||||
// _ => unreachable!(),
|
||||
// };
|
||||
|
||||
// tasks.push_back(tokio::task::spawn(async move {
|
||||
// let block = Block::consensus_decode(&mut Cursor::new(raw_block)).unwrap();
|
||||
|
||||
// (blk_metadata, block)
|
||||
// }));
|
||||
|
||||
// while tasks.len() > BOUND {
|
||||
// let (blk_metadata, block) = rt.block_on(tasks.pop_front().unwrap()).unwrap();
|
||||
|
||||
// if send_block
|
||||
// .send(BlkMetadataAndBlock::new(blk_metadata, block))
|
||||
// .is_err()
|
||||
// {
|
||||
// return ControlFlow::Break(());
|
||||
// }
|
||||
// }
|
||||
|
||||
// ControlFlow::Continue(())
|
||||
// });
|
||||
//
|
||||
// todo!("Send the rest")
|
||||
// });
|
||||
|
||||
thread::spawn(move || {
|
||||
let mut height = start_recap.map_or(0, |(_, recap)| recap.height());
|
||||
|
||||
let mut future_blocks = BTreeMap::default();
|
||||
let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default();
|
||||
let mut recent_hashes: BTreeSet<BlockHash> = BTreeSet::default();
|
||||
|
||||
let mut prev_hash =
|
||||
start_recap.map_or_else(BlockHash::all_zeros, |(_, recap)| *recap.prev_hash());
|
||||
|
||||
let mut prepare_and_send = |(hash, tuple): (BlockHash, BlkMetadataAndBlock)| {
|
||||
blk_index_to_blk_recap.update(&tuple, height);
|
||||
|
||||
if start.map_or(true, |start| start <= height) {
|
||||
send_height_block_hash
|
||||
.send((height, tuple.block, hash))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
if end.map_or(false, |end| height == end) {
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
|
||||
height += 1;
|
||||
|
||||
ControlFlow::Continue(())
|
||||
};
|
||||
|
||||
let mut update_tip = |prev_hash: &mut BlockHash,
|
||||
recent_hashes: &mut BTreeSet<BlockHash>,
|
||||
recent_chain: &mut VecDeque<(BlockHash, BlkMetadataAndBlock)>,
|
||||
future_blocks: &mut BTreeMap<BlockHash, BlkMetadataAndBlock>,
|
||||
tuple: BlkMetadataAndBlock| {
|
||||
let mut tuple = Some(tuple);
|
||||
|
||||
while let Some(tuple) = tuple.take().or_else(|| future_blocks.remove(prev_hash)) {
|
||||
let hash = tuple.block.block_hash();
|
||||
|
||||
*prev_hash = hash;
|
||||
recent_hashes.insert(hash);
|
||||
recent_chain.push_back((hash, tuple));
|
||||
}
|
||||
|
||||
while recent_chain.len() > NUMBER_OF_UNSAFE_BLOCKS {
|
||||
let (hash, tuple) = recent_chain.pop_front().unwrap();
|
||||
|
||||
recent_hashes.remove(&hash);
|
||||
|
||||
if prepare_and_send((hash, tuple)).is_break() {
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
};
|
||||
|
||||
let flow = recv_block.iter().try_for_each(|tuple| {
|
||||
// block isn't next after current tip
|
||||
if prev_hash != tuple.block.header.prev_blockhash {
|
||||
let is_block_active =
|
||||
|hash| rpc.get_block_header_info(hash).unwrap().confirmations > 0;
|
||||
|
||||
// block prev has already been processed
|
||||
if recent_hashes.contains(&tuple.block.header.prev_blockhash) {
|
||||
let hash = tuple.block.block_hash();
|
||||
|
||||
if is_block_active(&hash) {
|
||||
let prev_index = recent_chain
|
||||
.iter()
|
||||
.position(|(hash, ..)| hash == &tuple.block.header.prev_blockhash)
|
||||
.unwrap();
|
||||
|
||||
let bad_index_start = prev_index + 1;
|
||||
|
||||
recent_chain.drain(bad_index_start..).for_each(|(hash, _)| {
|
||||
recent_hashes.remove(&hash);
|
||||
});
|
||||
|
||||
return update_tip(
|
||||
&mut prev_hash,
|
||||
&mut recent_hashes,
|
||||
&mut recent_chain,
|
||||
&mut future_blocks,
|
||||
tuple,
|
||||
);
|
||||
}
|
||||
// Check if there was already a future block with the same prev hash
|
||||
} else if let Some(prev_tuple) =
|
||||
future_blocks.insert(tuple.block.header.prev_blockhash, tuple)
|
||||
{
|
||||
// If the previous was the active one
|
||||
if is_block_active(&prev_tuple.block.block_hash()) {
|
||||
// Rollback the insert
|
||||
future_blocks.insert(prev_tuple.block.header.prev_blockhash, prev_tuple);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return update_tip(
|
||||
&mut prev_hash,
|
||||
&mut recent_hashes,
|
||||
&mut recent_chain,
|
||||
&mut future_blocks,
|
||||
tuple,
|
||||
);
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
|
||||
if flow.is_continue() {
|
||||
// Send the last (up to 100) blocks
|
||||
recent_chain.into_iter().try_for_each(prepare_and_send);
|
||||
}
|
||||
|
||||
blk_index_to_blk_recap.export();
|
||||
});
|
||||
|
||||
recv_height_block_hash
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
use bitcoincore_rpc::{Auth, Client};
|
||||
|
||||
fn main() {
|
||||
let i = std::time::Instant::now();
|
||||
|
||||
let url = "http://localhost:8332";
|
||||
let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
let rpc = Client::new(url, auth).unwrap();
|
||||
|
||||
let data_dir = "../bitcoin";
|
||||
let export_dir = "./target";
|
||||
let start = None;
|
||||
let end = None;
|
||||
|
||||
biter::new(data_dir, export_dir, start, end, rpc)
|
||||
.iter()
|
||||
.for_each(|(height, _block, hash)| {
|
||||
println!("{height}: {hash}");
|
||||
});
|
||||
|
||||
dbg!(i.elapsed());
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
use std::{collections::BTreeMap, fs, path::PathBuf, time::UNIX_EPOCH};
|
||||
|
||||
const BLK: &str = "blk";
|
||||
const DAT: &str = ".dat";
|
||||
|
||||
pub fn scan_blocks_dir(data_dir_path: &str) -> BTreeMap<usize, PathBuf> {
|
||||
let blocks_dir_path = &format!("{data_dir_path}/blocks");
|
||||
|
||||
fs::read_dir(blocks_dir_path)
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.filter(|path| {
|
||||
let is_file = path.is_file();
|
||||
|
||||
if is_file {
|
||||
let file_name = path.file_name().unwrap().to_str().unwrap();
|
||||
|
||||
file_name.starts_with(BLK) && file_name.ends_with(DAT)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
.map(|path| {
|
||||
let file_name = path.file_name().unwrap().to_str().unwrap();
|
||||
|
||||
let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())]
|
||||
.parse::<usize>()
|
||||
.unwrap();
|
||||
|
||||
(blk_index, path)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>()
|
||||
}
|
||||
|
||||
pub fn path_to_modified_time(path: &PathBuf) -> u64 {
|
||||
fs::metadata(path)
|
||||
.unwrap()
|
||||
.modified()
|
||||
.unwrap()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
}
|
||||
Generated
+1
-3
@@ -274,9 +274,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "biter"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd2ea6b4c4046feaaf73f3dff1cc648328af021d50be95946e0e7a807c985b47"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"bitcoincore-rpc",
|
||||
|
||||
+2
-1
@@ -10,7 +10,8 @@ allocative = "0.3.3"
|
||||
bincode = { git = "https://github.com/bincode-org/bincode.git", features = [
|
||||
"serde",
|
||||
] }
|
||||
biter = "0.1.0"
|
||||
# biter = "0.1.0"
|
||||
biter = { path = "../biter" }
|
||||
bitcoin_hashes = { version = "0.14.0" }
|
||||
chrono = { version = "0.4.38", features = ["serde"] }
|
||||
clap = { version = "4.5.19", features = ["derive"] }
|
||||
|
||||
+1
-1
@@ -2,6 +2,6 @@
|
||||
|
||||
echo "Deleting datasets..."
|
||||
rm -r ../datasets
|
||||
echo "Deleting states and datasets..."
|
||||
echo "Deleting states and databases..."
|
||||
rm -r ./out
|
||||
echo "Done."
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::{
|
||||
datasets::{AllDatasets, ComputeData},
|
||||
io::OUTPUTS_FOLDER_PATH,
|
||||
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
|
||||
structs::{Date, DateData, MapKey},
|
||||
structs::{DateData, MapKey, Timestamp},
|
||||
utils::{generate_allocation_files, log, time},
|
||||
Config, Exit, Height,
|
||||
};
|
||||
@@ -81,9 +81,9 @@ pub fn iter_blocks(
|
||||
if let Some((_current_block_height, current_block, _current_block_hash)) =
|
||||
current_block_opt
|
||||
{
|
||||
let timestamp = current_block.header.time;
|
||||
let timestamp = Timestamp::wrap(current_block.header.time);
|
||||
|
||||
let current_block_date = Date::from_timestamp(timestamp);
|
||||
let current_block_date = timestamp.to_date();
|
||||
let current_block_height: Height = height + blocks_loop_i;
|
||||
|
||||
if current_block_height.to_usize() != _current_block_height {
|
||||
@@ -91,9 +91,9 @@ pub fn iter_blocks(
|
||||
panic!()
|
||||
}
|
||||
|
||||
let next_block_date = next_block_opt
|
||||
.as_ref()
|
||||
.map(|(_, next_block, _)| Date::from_timestamp(next_block.header.time));
|
||||
let next_block_date = next_block_opt.as_ref().map(|(_, next_block, _)| {
|
||||
Timestamp::wrap(next_block.header.time).to_date()
|
||||
});
|
||||
|
||||
// Always run for the first block of the loop
|
||||
if blocks_loop_date.is_none() {
|
||||
@@ -168,7 +168,6 @@ pub fn iter_blocks(
|
||||
height: current_block_height,
|
||||
is_date_last_block,
|
||||
states: &mut states,
|
||||
timestamp,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -213,7 +212,7 @@ pub fn iter_blocks(
|
||||
});
|
||||
}
|
||||
|
||||
if !config.dry_run {
|
||||
if !config.dry_run() {
|
||||
let is_safe = height.is_safe(approx_block_count);
|
||||
|
||||
export(ExportedData {
|
||||
@@ -225,7 +224,7 @@ pub fn iter_blocks(
|
||||
exit: exit.clone(),
|
||||
})?;
|
||||
|
||||
if config.record_ram_usage {
|
||||
if config.record_ram_usage() {
|
||||
time("Exporing allocation files", || {
|
||||
generate_allocation_files(&datasets, &databases, &states, last_height)
|
||||
})?;
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::{
|
||||
},
|
||||
structs::{
|
||||
Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Counter, Date,
|
||||
EmptyAddressData, Height, PartialTxoutData, Price, SentData, TxData, TxoutIndex,
|
||||
EmptyAddressData, Height, PartialTxoutData, Price, SentData, Timestamp, TxData, TxoutIndex,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -37,7 +37,6 @@ pub struct ParseData<'a> {
|
||||
pub is_date_last_block: bool,
|
||||
pub rpc: &'a biter::bitcoincore_rpc::Client,
|
||||
pub states: &'a mut States,
|
||||
pub timestamp: u32,
|
||||
}
|
||||
|
||||
pub fn parse(
|
||||
@@ -53,11 +52,12 @@ pub fn parse(
|
||||
is_date_last_block,
|
||||
rpc,
|
||||
states,
|
||||
timestamp,
|
||||
}: ParseData,
|
||||
) {
|
||||
// log(&format!("{height}"));
|
||||
|
||||
let timestamp = Timestamp::wrap(block.header.time);
|
||||
|
||||
// If false, expect that the code is flawless
|
||||
// or create a 0 value txid database
|
||||
let enable_check_if_txout_value_is_zero_in_db: bool = true;
|
||||
@@ -89,9 +89,9 @@ pub fn parse(
|
||||
let block_size = block.total_size();
|
||||
let block_weight = block.weight().to_wu();
|
||||
let block_vbytes = block.weight().to_vbytes_floor();
|
||||
let block_interval = previous_timestamp.map_or(0, |previous_timestamp| {
|
||||
let block_interval = previous_timestamp.map_or(Timestamp::ZERO, |previous_timestamp| {
|
||||
if previous_timestamp >= timestamp {
|
||||
0
|
||||
Timestamp::ZERO
|
||||
} else {
|
||||
timestamp - previous_timestamp
|
||||
}
|
||||
@@ -519,6 +519,8 @@ pub fn parse(
|
||||
input_amount,
|
||||
block_price,
|
||||
previous_price,
|
||||
timestamp,
|
||||
input_block_data.timestamp,
|
||||
);
|
||||
};
|
||||
|
||||
@@ -645,6 +647,7 @@ pub fn parse(
|
||||
&states.date_data_vec,
|
||||
&block_path_to_sent_data,
|
||||
block_price,
|
||||
timestamp,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use allocative::Allocative;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
structs::{AnyHeightMap, Date, HeightMap},
|
||||
structs::{AnyHeightMap, Date, HeightMap, Timestamp},
|
||||
};
|
||||
|
||||
use super::{InsertData, MinInitialStates};
|
||||
@@ -13,7 +13,7 @@ pub struct BlockMetadataDataset {
|
||||
|
||||
// Inserted
|
||||
pub date: HeightMap<Date>,
|
||||
pub timestamp: HeightMap<u32>,
|
||||
pub timestamp: HeightMap<Timestamp>,
|
||||
}
|
||||
|
||||
impl BlockMetadataDataset {
|
||||
@@ -41,7 +41,7 @@ impl BlockMetadataDataset {
|
||||
) {
|
||||
self.timestamp.insert(height, timestamp);
|
||||
|
||||
self.date.insert(height, Date::from_timestamp(timestamp));
|
||||
self.date.insert(height, timestamp.to_date());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -302,7 +302,7 @@ impl MiningDataset {
|
||||
self.block_weight
|
||||
.insert(height, block_weight as f32 / BYTES_IN_MB as f32);
|
||||
self.block_vbytes.insert(height, block_vbytes);
|
||||
self.block_interval.insert(height, block_interval);
|
||||
self.block_interval.insert(height, *block_interval);
|
||||
|
||||
if is_date_last_block {
|
||||
self.coinbase.date_insert_sum_range(date, date_blocks_range);
|
||||
|
||||
@@ -45,7 +45,7 @@ use crate::{
|
||||
// UTXOCohortsReceivedStates,
|
||||
UTXOCohortsSentStates,
|
||||
},
|
||||
structs::{Amount, Date, Height, Price},
|
||||
structs::{Amount, Date, Height, Price, Timestamp},
|
||||
};
|
||||
|
||||
pub struct InsertData<'a> {
|
||||
@@ -53,7 +53,7 @@ pub struct InsertData<'a> {
|
||||
pub address_cohorts_one_shot_states: &'a Option<AddressCohortsOneShotStates>,
|
||||
pub address_cohorts_realized_states: &'a Option<AddressCohortsRealizedStates>,
|
||||
pub amount_sent: Amount,
|
||||
pub block_interval: u32,
|
||||
pub block_interval: Timestamp,
|
||||
pub block_price: Price,
|
||||
pub block_size: usize,
|
||||
pub block_vbytes: u64,
|
||||
@@ -71,7 +71,7 @@ pub struct InsertData<'a> {
|
||||
pub satblocks_destroyed: Amount,
|
||||
pub satdays_destroyed: Amount,
|
||||
pub states: &'a States,
|
||||
pub timestamp: u32,
|
||||
pub timestamp: Timestamp,
|
||||
pub transaction_count: usize,
|
||||
pub utxo_cohorts_one_shot_states: &'a UTXOCohortsOneShotStates,
|
||||
// pub utxo_cohorts_received_states: &'a UTXOCohortsReceivedStates,
|
||||
|
||||
@@ -3,17 +3,16 @@ mod ohlc;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use allocative::Allocative;
|
||||
use chrono::{Days, NaiveDateTime, NaiveTime, TimeZone, Timelike, Utc};
|
||||
use chrono::Days;
|
||||
use color_eyre::eyre::Error;
|
||||
|
||||
pub use ohlc::*;
|
||||
|
||||
use crate::{
|
||||
log,
|
||||
price::{Binance, Kibo, Kraken},
|
||||
structs::{
|
||||
AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId,
|
||||
MapKey,
|
||||
Amount, AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height,
|
||||
HeightMapChunkId, MapKey, Timestamp,
|
||||
},
|
||||
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
|
||||
};
|
||||
@@ -76,9 +75,11 @@ pub struct PriceDatasets {
|
||||
pub price_10y_total_return: DateMap<f32>,
|
||||
pub price_4y_compound_return: DateMap<f32>,
|
||||
// projection via lowest 4y compound value
|
||||
pub all_time_high: BiMap<f32>,
|
||||
pub market_price_to_all_time_high_ratio: BiMap<f32>,
|
||||
pub drawdown: BiMap<f32>,
|
||||
pub sats_per_dollar: BiMap<f32>,
|
||||
// volatility
|
||||
// drawdown
|
||||
// sats per dollar
|
||||
}
|
||||
|
||||
impl PriceDatasets {
|
||||
@@ -138,6 +139,13 @@ impl PriceDatasets {
|
||||
price_8y_total_return: DateMap::new_bin(1, &f("price_8y_total_return")),
|
||||
price_10y_total_return: DateMap::new_bin(1, &f("price_10y_total_return")),
|
||||
price_4y_compound_return: DateMap::new_bin(1, &f("price_4y_compound_return")),
|
||||
all_time_high: BiMap::new_bin(1, &f("all_time_high")),
|
||||
market_price_to_all_time_high_ratio: BiMap::new_bin(
|
||||
1,
|
||||
&f("market_price_to_all_time_high_ratio"),
|
||||
),
|
||||
drawdown: BiMap::new_bin(1, &f("drawdown")),
|
||||
sats_per_dollar: BiMap::new_bin(1, &f("sats_per_dollar")),
|
||||
};
|
||||
|
||||
s.min_initial_states
|
||||
@@ -306,6 +314,26 @@ impl PriceDatasets {
|
||||
.compute(compute_data, &mut self.closes, &mut self.price_144d_sma);
|
||||
self.price_200w_sma_ratio
|
||||
.compute(compute_data, &mut self.closes, &mut self.price_200w_sma);
|
||||
|
||||
self.all_time_high
|
||||
.multi_insert_max(heights, dates, &mut self.closes);
|
||||
|
||||
self.market_price_to_all_time_high_ratio
|
||||
.multi_insert_percentage(heights, dates, &mut self.closes, &mut self.all_time_high);
|
||||
|
||||
self.drawdown.multi_insert_simple_transform(
|
||||
heights,
|
||||
dates,
|
||||
&mut self.market_price_to_all_time_high_ratio,
|
||||
&|v| -(100.0 - v),
|
||||
);
|
||||
|
||||
self.sats_per_dollar.multi_insert_simple_transform(
|
||||
heights,
|
||||
dates,
|
||||
&mut self.closes,
|
||||
&|price| Amount::ONE_BTC_F32 / price,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result<OHLC> {
|
||||
@@ -396,31 +424,20 @@ impl PriceDatasets {
|
||||
pub fn get_height_ohlc(
|
||||
&mut self,
|
||||
height: Height,
|
||||
timestamp: u32,
|
||||
previous_timestamp: Option<u32>,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if let Some(ohlc) = self.ohlcs.height.get(&height) {
|
||||
return Ok(ohlc);
|
||||
}
|
||||
|
||||
let clean_timestamp = |timestamp| {
|
||||
let date_time = Utc.timestamp_opt(i64::from(timestamp), 0).unwrap();
|
||||
|
||||
NaiveDateTime::new(
|
||||
date_time.date_naive(),
|
||||
NaiveTime::from_hms_opt(date_time.hour(), date_time.minute(), 0).unwrap(),
|
||||
)
|
||||
.and_utc()
|
||||
.timestamp() as u32
|
||||
};
|
||||
|
||||
let timestamp = clean_timestamp(timestamp);
|
||||
let timestamp = timestamp.to_floored_seconds();
|
||||
|
||||
if previous_timestamp.is_none() && !height.is_first() {
|
||||
panic!("Shouldn't be possible");
|
||||
}
|
||||
|
||||
let previous_timestamp = previous_timestamp.map(clean_timestamp);
|
||||
let previous_timestamp = previous_timestamp.map(|t| t.to_floored_seconds());
|
||||
|
||||
let ohlc = self
|
||||
.get_from_1mn_kraken(timestamp, previous_timestamp)
|
||||
@@ -430,7 +447,7 @@ impl PriceDatasets {
|
||||
self.get_from_har_binance(timestamp, previous_timestamp)
|
||||
.unwrap_or_else(|_| {
|
||||
self.get_from_height_kibo(&height).unwrap_or_else(|_| {
|
||||
let date = Date::from_timestamp(timestamp);
|
||||
let date = timestamp.to_date();
|
||||
|
||||
panic!(
|
||||
"Can't find the price for: height: {height} - date: {date}
|
||||
@@ -479,8 +496,8 @@ How to fix this:
|
||||
|
||||
fn get_from_1mn_kraken(
|
||||
&mut self,
|
||||
timestamp: u32,
|
||||
previous_timestamp: Option<u32>,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if self.kraken_1mn.is_none()
|
||||
|| self
|
||||
@@ -500,8 +517,8 @@ How to fix this:
|
||||
|
||||
fn get_from_1mn_binance(
|
||||
&mut self,
|
||||
timestamp: u32,
|
||||
previous_timestamp: Option<u32>,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if self.binance_1mn.is_none()
|
||||
|| self
|
||||
@@ -526,8 +543,8 @@ How to fix this:
|
||||
|
||||
fn get_from_har_binance(
|
||||
&mut self,
|
||||
timestamp: u32,
|
||||
previous_timestamp: Option<u32>,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if self.binance_har.is_none() {
|
||||
self.binance_har
|
||||
@@ -544,8 +561,8 @@ How to fix this:
|
||||
|
||||
fn find_height_ohlc(
|
||||
tree: &Option<BTreeMap<u32, OHLC>>,
|
||||
timestamp: u32,
|
||||
previous_timestamp: Option<u32>,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
name: &str,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
let tree = tree.as_ref().unwrap();
|
||||
@@ -572,12 +589,12 @@ How to fix this:
|
||||
close: previous_ohlc.close,
|
||||
};
|
||||
|
||||
let start = previous_timestamp.unwrap_or(0);
|
||||
let start = previous_timestamp.unwrap_or_default();
|
||||
let end = timestamp;
|
||||
|
||||
// Otherwise it's a re-org
|
||||
if start < end {
|
||||
tree.range(&start..=&end).skip(1).for_each(|(_, ohlc)| {
|
||||
tree.range(&*start..=&*end).skip(1).for_each(|(_, ohlc)| {
|
||||
if ohlc.high > final_ohlc.high {
|
||||
final_ohlc.high = ohlc.high
|
||||
}
|
||||
@@ -624,6 +641,10 @@ impl AnyDataset for PriceDatasets {
|
||||
&self.price_89d_sma,
|
||||
&self.price_144d_sma,
|
||||
&self.price_200w_sma,
|
||||
&self.all_time_high,
|
||||
&self.market_price_to_all_time_high_ratio,
|
||||
&self.drawdown,
|
||||
&self.sats_per_dollar,
|
||||
];
|
||||
|
||||
v.append(&mut self.price_1w_sma_ratio.to_computed_bi_map_vec());
|
||||
@@ -660,6 +681,10 @@ impl AnyDataset for PriceDatasets {
|
||||
&mut self.price_89d_sma,
|
||||
&mut self.price_144d_sma,
|
||||
&mut self.price_200w_sma,
|
||||
&mut self.all_time_high,
|
||||
&mut self.market_price_to_all_time_high_ratio,
|
||||
&mut self.drawdown,
|
||||
&mut self.sats_per_dollar,
|
||||
];
|
||||
|
||||
v.append(&mut self.price_1w_sma_ratio.to_computed_mut_bi_map_vec());
|
||||
|
||||
@@ -66,7 +66,7 @@ impl CapitalizationDataset {
|
||||
let realized_cap = self
|
||||
.realized_cap
|
||||
.height
|
||||
.insert(height, state.realized_cap.to_dollar() as f32);
|
||||
.insert(height, state.realized_cap().to_dollar() as f32);
|
||||
|
||||
if is_date_last_block {
|
||||
self.realized_cap.date.insert(date, realized_cap);
|
||||
|
||||
@@ -51,9 +51,12 @@ impl InputSubDataset {
|
||||
}: &InsertData,
|
||||
state: &InputState,
|
||||
) {
|
||||
let count = self.count.height.insert(height, state.count.round() as u64);
|
||||
let count = self
|
||||
.count
|
||||
.height
|
||||
.insert(height, state.count().round() as u64);
|
||||
|
||||
self.volume.height.insert(height, state.volume.to_btc());
|
||||
self.volume.height.insert(height, state.volume().to_btc());
|
||||
|
||||
if is_date_last_block {
|
||||
self.count.date.insert(date, count);
|
||||
|
||||
@@ -83,28 +83,25 @@ impl PricePaidSubDataset {
|
||||
}: &InsertData,
|
||||
state: &PricePaidState,
|
||||
) {
|
||||
let PricePaidState {
|
||||
pp_05p,
|
||||
pp_10p,
|
||||
pp_15p,
|
||||
pp_20p,
|
||||
pp_25p,
|
||||
pp_30p,
|
||||
pp_35p,
|
||||
pp_40p,
|
||||
pp_45p,
|
||||
pp_median,
|
||||
pp_55p,
|
||||
pp_60p,
|
||||
pp_65p,
|
||||
pp_70p,
|
||||
pp_75p,
|
||||
pp_80p,
|
||||
pp_85p,
|
||||
pp_90p,
|
||||
pp_95p,
|
||||
..
|
||||
} = state;
|
||||
let pp_05p = state.pp_05p();
|
||||
let pp_10p = state.pp_10p();
|
||||
let pp_15p = state.pp_15p();
|
||||
let pp_20p = state.pp_20p();
|
||||
let pp_25p = state.pp_25p();
|
||||
let pp_30p = state.pp_30p();
|
||||
let pp_35p = state.pp_35p();
|
||||
let pp_40p = state.pp_40p();
|
||||
let pp_45p = state.pp_45p();
|
||||
let pp_median = state.pp_median();
|
||||
let pp_55p = state.pp_55p();
|
||||
let pp_60p = state.pp_60p();
|
||||
let pp_65p = state.pp_65p();
|
||||
let pp_70p = state.pp_70p();
|
||||
let pp_75p = state.pp_75p();
|
||||
let pp_80p = state.pp_80p();
|
||||
let pp_85p = state.pp_85p();
|
||||
let pp_90p = state.pp_90p();
|
||||
let pp_95p = state.pp_95p();
|
||||
|
||||
// Check if iter was empty
|
||||
if pp_05p.is_none() {
|
||||
|
||||
@@ -15,9 +15,12 @@ pub struct RealizedSubDataset {
|
||||
// Inserted
|
||||
realized_profit: BiMap<f32>,
|
||||
realized_loss: BiMap<f32>,
|
||||
value_destroyed: BiMap<f32>,
|
||||
value_created: BiMap<f32>,
|
||||
adjusted_value_created: BiMap<f32>,
|
||||
value_destroyed: BiMap<f32>,
|
||||
adjusted_value_destroyed: BiMap<f32>,
|
||||
spent_output_profit_ratio: BiMap<f32>,
|
||||
adjusted_spent_output_profit_ratio: BiMap<f32>,
|
||||
|
||||
// Computed
|
||||
negative_realized_loss: BiMap<f32>,
|
||||
@@ -47,8 +50,14 @@ impl RealizedSubDataset {
|
||||
realized_profit: BiMap::new_bin(1, &f("realized_profit")),
|
||||
realized_loss: BiMap::new_bin(1, &f("realized_loss")),
|
||||
value_created: BiMap::new_bin(1, &f("value_created")),
|
||||
adjusted_value_created: BiMap::new_bin(1, &f("adjusted_value_created")),
|
||||
value_destroyed: BiMap::new_bin(1, &f("value_destroyed")),
|
||||
adjusted_value_destroyed: BiMap::new_bin(1, &f("adjusted_value_destroyed")),
|
||||
spent_output_profit_ratio: BiMap::new_bin(2, &f("spent_output_profit_ratio")),
|
||||
adjusted_spent_output_profit_ratio: BiMap::new_bin(
|
||||
2,
|
||||
&f("adjusted_spent_output_profit_ratio"),
|
||||
),
|
||||
|
||||
negative_realized_loss: BiMap::new_bin(2, &f("negative_realized_loss")),
|
||||
net_realized_profit_and_loss: BiMap::new_bin(1, &f("net_realized_profit_and_loss")),
|
||||
@@ -89,29 +98,51 @@ impl RealizedSubDataset {
|
||||
) {
|
||||
self.realized_profit
|
||||
.height
|
||||
.insert(height, height_state.realized_profit.to_dollar() as f32);
|
||||
.insert(height, height_state.realized_profit().to_dollar() as f32);
|
||||
|
||||
self.realized_loss
|
||||
.height
|
||||
.insert(height, height_state.realized_loss.to_dollar() as f32);
|
||||
.insert(height, height_state.realized_loss().to_dollar() as f32);
|
||||
|
||||
self.value_created
|
||||
.height
|
||||
.insert(height, height_state.value_created.to_dollar() as f32);
|
||||
.insert(height, height_state.value_created().to_dollar() as f32);
|
||||
|
||||
self.adjusted_value_created.height.insert(
|
||||
height,
|
||||
height_state.adjusted_value_created().to_dollar() as f32,
|
||||
);
|
||||
|
||||
self.value_destroyed
|
||||
.height
|
||||
.insert(height, height_state.value_destroyed.to_dollar() as f32);
|
||||
.insert(height, height_state.value_destroyed().to_dollar() as f32);
|
||||
|
||||
self.adjusted_value_destroyed.height.insert(
|
||||
height,
|
||||
height_state.adjusted_value_destroyed().to_dollar() as f32,
|
||||
);
|
||||
|
||||
self.spent_output_profit_ratio.height.insert(height, {
|
||||
if height_state.value_destroyed > Price::ZERO {
|
||||
(height_state.value_created.to_cent() as f64
|
||||
/ height_state.value_destroyed.to_cent() as f64) as f32
|
||||
if height_state.value_destroyed() > Price::ZERO {
|
||||
(height_state.value_created().to_cent() as f64
|
||||
/ height_state.value_destroyed().to_cent() as f64) as f32
|
||||
} else {
|
||||
1.0
|
||||
}
|
||||
});
|
||||
|
||||
self.adjusted_spent_output_profit_ratio
|
||||
.height
|
||||
.insert(height, {
|
||||
if height_state.adjusted_value_destroyed() > Price::ZERO {
|
||||
(height_state.adjusted_value_created().to_cent() as f64
|
||||
/ height_state.adjusted_value_destroyed().to_cent() as f64)
|
||||
as f32
|
||||
} else {
|
||||
1.0
|
||||
}
|
||||
});
|
||||
|
||||
if is_date_last_block {
|
||||
self.realized_profit
|
||||
.date_insert_sum_range(date, date_blocks_range);
|
||||
@@ -122,14 +153,31 @@ impl RealizedSubDataset {
|
||||
self.value_created
|
||||
.date_insert_sum_range(date, date_blocks_range);
|
||||
|
||||
self.adjusted_value_created
|
||||
.date_insert_sum_range(date, date_blocks_range);
|
||||
|
||||
self.value_destroyed
|
||||
.date_insert_sum_range(date, date_blocks_range);
|
||||
|
||||
self.adjusted_value_destroyed
|
||||
.date_insert_sum_range(date, date_blocks_range);
|
||||
|
||||
self.spent_output_profit_ratio.date.insert(
|
||||
date,
|
||||
self.value_created.height.sum_range(date_blocks_range)
|
||||
/ self.value_destroyed.height.sum_range(date_blocks_range),
|
||||
);
|
||||
|
||||
self.adjusted_spent_output_profit_ratio.date.insert(
|
||||
date,
|
||||
self.adjusted_value_created
|
||||
.height
|
||||
.sum_range(date_blocks_range)
|
||||
/ self
|
||||
.adjusted_value_destroyed
|
||||
.height
|
||||
.sum_range(date_blocks_range),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,8 +256,11 @@ impl AnyDataset for RealizedSubDataset {
|
||||
&self.realized_loss,
|
||||
&self.realized_profit,
|
||||
&self.value_created,
|
||||
&self.adjusted_value_created,
|
||||
&self.value_destroyed,
|
||||
&self.adjusted_value_destroyed,
|
||||
&self.spent_output_profit_ratio,
|
||||
&self.adjusted_spent_output_profit_ratio,
|
||||
]
|
||||
}
|
||||
|
||||
@@ -218,8 +269,11 @@ impl AnyDataset for RealizedSubDataset {
|
||||
&mut self.realized_loss,
|
||||
&mut self.realized_profit,
|
||||
&mut self.value_created,
|
||||
&mut self.adjusted_value_created,
|
||||
&mut self.value_destroyed,
|
||||
&mut self.adjusted_value_destroyed,
|
||||
&mut self.spent_output_profit_ratio,
|
||||
&mut self.adjusted_spent_output_profit_ratio,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@@ -254,99 +254,3 @@ where
|
||||
v
|
||||
}
|
||||
}
|
||||
|
||||
// impl<Key, Value, ChunkId, Serialized> AnyDataset for RecapDataset<Key, Value, ChunkId, Serialized>
|
||||
// where
|
||||
// Value: MapValue,
|
||||
// ChunkId: MapChunkId,
|
||||
// Key: MapKey<ChunkId>,
|
||||
// Serialized: MapSerialized<Key, Value, ChunkId>,
|
||||
// {
|
||||
// fn get_min_initial_states(&self) -> &MinInitialStates {
|
||||
// &self.min_initial_states
|
||||
// }
|
||||
|
||||
// fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
|
||||
// let mut v: Vec<&(dyn AnyDateMap + Send + Sync)> = vec![];
|
||||
|
||||
// if let Some(min) = self.min.as_ref() {
|
||||
// v.push(min);
|
||||
// }
|
||||
|
||||
// if let Some(max) = self.max.as_ref() {
|
||||
// v.push(max);
|
||||
// }
|
||||
|
||||
// if let Some(median) = self.median.as_ref() {
|
||||
// v.push(median);
|
||||
// }
|
||||
|
||||
// if let Some(average) = self.average.as_ref() {
|
||||
// v.push(average);
|
||||
// }
|
||||
|
||||
// if let Some(sum) = self.sum.as_ref() {
|
||||
// v.push(sum);
|
||||
// }
|
||||
|
||||
// if let Some(_90p) = self._90p.as_ref() {
|
||||
// v.push(_90p);
|
||||
// }
|
||||
|
||||
// if let Some(_75p) = self._75p.as_ref() {
|
||||
// v.push(_75p);
|
||||
// }
|
||||
|
||||
// if let Some(_25p) = self._25p.as_ref() {
|
||||
// v.push(_25p);
|
||||
// }
|
||||
|
||||
// if let Some(_10p) = self._10p.as_ref() {
|
||||
// v.push(_10p);
|
||||
// }
|
||||
|
||||
// v
|
||||
// }
|
||||
|
||||
// fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
|
||||
// let mut v: Vec<&mut dyn AnyDateMap> = vec![];
|
||||
|
||||
// if let Some(min) = self.min.as_mut() {
|
||||
// v.push(min);
|
||||
// }
|
||||
|
||||
// if let Some(max) = self.max.as_mut() {
|
||||
// v.push(max);
|
||||
// }
|
||||
|
||||
// if let Some(median) = self.median.as_mut() {
|
||||
// v.push(median);
|
||||
// }
|
||||
|
||||
// if let Some(average) = self.average.as_mut() {
|
||||
// v.push(average);
|
||||
// }
|
||||
|
||||
// if let Some(sum) = self.sum.as_mut() {
|
||||
// v.push(sum);
|
||||
// }
|
||||
|
||||
// if let Some(_90p) = self._90p.as_mut() {
|
||||
// v.push(_90p);
|
||||
// }
|
||||
|
||||
// if let Some(_75p) = self._75p.as_mut() {
|
||||
// v.push(_75p);
|
||||
// }
|
||||
|
||||
// if let Some(_25p) = self._25p.as_mut() {
|
||||
// v.push(_25p);
|
||||
// }
|
||||
|
||||
// if let Some(_10p) = self._10p.as_mut() {
|
||||
// v.push(_10p);
|
||||
// }
|
||||
|
||||
// v
|
||||
// }
|
||||
// }
|
||||
|
||||
@@ -60,7 +60,7 @@ impl SupplySubDataset {
|
||||
}: &InsertData,
|
||||
state: &SupplyState,
|
||||
) {
|
||||
let total_supply = self.supply.height.insert(height, state.supply.to_btc());
|
||||
let total_supply = self.supply.height.insert(height, state.supply().to_btc());
|
||||
|
||||
if is_date_last_block {
|
||||
self.supply.date.insert(date, total_supply);
|
||||
|
||||
@@ -86,30 +86,30 @@ impl UnrealizedSubDataset {
|
||||
) {
|
||||
self.supply_in_profit
|
||||
.height
|
||||
.insert(height, block_state.supply_in_profit.to_btc());
|
||||
.insert(height, block_state.supply_in_profit().to_btc());
|
||||
|
||||
self.unrealized_profit
|
||||
.height
|
||||
.insert(height, block_state.unrealized_profit.to_dollar() as f32);
|
||||
.insert(height, block_state.unrealized_profit().to_dollar() as f32);
|
||||
|
||||
self.unrealized_loss
|
||||
.height
|
||||
.insert(height, block_state.unrealized_loss.to_dollar() as f32);
|
||||
.insert(height, block_state.unrealized_loss().to_dollar() as f32);
|
||||
|
||||
if is_date_last_block {
|
||||
let date_state = date_state.as_ref().unwrap();
|
||||
|
||||
self.supply_in_profit
|
||||
.date
|
||||
.insert(date, date_state.supply_in_profit.to_btc());
|
||||
.insert(date, date_state.supply_in_profit().to_btc());
|
||||
|
||||
self.unrealized_profit
|
||||
.date
|
||||
.insert(date, date_state.unrealized_profit.to_dollar() as f32);
|
||||
.insert(date, date_state.unrealized_profit().to_dollar() as f32);
|
||||
|
||||
self.unrealized_loss
|
||||
.date
|
||||
.insert(date, date_state.unrealized_loss.to_dollar() as f32);
|
||||
.insert(date, date_state.unrealized_loss().to_dollar() as f32);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ impl UTXOSubDataset {
|
||||
}: &InsertData,
|
||||
state: &UTXOState,
|
||||
) {
|
||||
let count = self.count.height.insert(height, state.count);
|
||||
let count = self.count.height.insert(height, state.count());
|
||||
|
||||
if is_date_last_block {
|
||||
self.count.date.insert(date, count);
|
||||
|
||||
@@ -9,7 +9,7 @@ use serde_json::Value;
|
||||
use crate::{
|
||||
datasets::OHLC,
|
||||
io::{Json, INPUTS_FOLDER_PATH},
|
||||
structs::Date,
|
||||
structs::{Date, Timestamp},
|
||||
utils::{log, retry},
|
||||
};
|
||||
|
||||
@@ -169,9 +169,10 @@ impl Binance {
|
||||
// [timestamp, open, high, low, close, volume, ...]
|
||||
let array = value.as_array().unwrap();
|
||||
|
||||
let date = Date::from_timestamp(
|
||||
let date = Timestamp::wrap(
|
||||
(array.first().unwrap().as_u64().unwrap() / 1_000) as u32,
|
||||
);
|
||||
)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
|
||||
@@ -5,7 +5,7 @@ use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
datasets::OHLC,
|
||||
structs::Date,
|
||||
structs::{Date, Timestamp},
|
||||
utils::{log, retry},
|
||||
};
|
||||
|
||||
@@ -91,8 +91,8 @@ impl Kraken {
|
||||
.map(|value| {
|
||||
let array = value.as_array().unwrap();
|
||||
|
||||
let date =
|
||||
Date::from_timestamp(array.first().unwrap().as_u64().unwrap() as u32);
|
||||
let date = Timestamp::wrap(array.first().unwrap().as_u64().unwrap() as u32)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
|
||||
@@ -93,7 +93,7 @@ impl AddressCohortDurableStates {
|
||||
|
||||
let one_shot_states_ref = &mut one_shot_states;
|
||||
|
||||
let supply = self.durable_states.supply_state.supply;
|
||||
let supply = self.durable_states.supply_state.supply();
|
||||
|
||||
self.price_to_amount.iterate(supply, |price_paid, amount| {
|
||||
one_shot_states_ref
|
||||
|
||||
@@ -19,14 +19,18 @@ impl AddressCohortsRealizedStates {
|
||||
let realized_profit = realized_data.profit;
|
||||
let realized_loss = realized_data.loss;
|
||||
let value_created = realized_data.value_created;
|
||||
let adjusted_value_created = realized_data.adjusted_value_created;
|
||||
let value_destroyed = realized_data.value_destroyed;
|
||||
let adjusted_value_destroyed = realized_data.adjusted_value_destroyed;
|
||||
|
||||
let normal_iteration = move |state: &mut RealizedState| -> color_eyre::Result<()> {
|
||||
state.iterate(
|
||||
realized_profit,
|
||||
realized_loss,
|
||||
value_created,
|
||||
adjusted_value_created,
|
||||
value_destroyed,
|
||||
adjusted_value_destroyed,
|
||||
);
|
||||
Ok(())
|
||||
};
|
||||
@@ -35,8 +39,12 @@ impl AddressCohortsRealizedStates {
|
||||
liquidity_classification.split(realized_profit.to_cent() as f64);
|
||||
let split_realized_loss = liquidity_classification.split(realized_loss.to_cent() as f64);
|
||||
let split_value_created = liquidity_classification.split(value_created.to_cent() as f64);
|
||||
let split_adjusted_value_created =
|
||||
liquidity_classification.split(adjusted_value_created.to_cent() as f64);
|
||||
let split_value_destroyed =
|
||||
liquidity_classification.split(value_destroyed.to_cent() as f64);
|
||||
let split_adjusted_value_destroyed =
|
||||
liquidity_classification.split(adjusted_value_destroyed.to_cent() as f64);
|
||||
|
||||
let liquified_iteration =
|
||||
move |liquidity, state: &mut RealizedState| -> color_eyre::Result<()> {
|
||||
@@ -44,7 +52,9 @@ impl AddressCohortsRealizedStates {
|
||||
Price::from_cent(split_realized_profit.from(liquidity) as u64),
|
||||
Price::from_cent(split_realized_loss.from(liquidity) as u64),
|
||||
Price::from_cent(split_value_created.from(liquidity) as u64),
|
||||
Price::from_cent(split_adjusted_value_created.from(liquidity) as u64),
|
||||
Price::from_cent(split_value_destroyed.from(liquidity) as u64),
|
||||
Price::from_cent(split_adjusted_value_destroyed.from(liquidity) as u64),
|
||||
);
|
||||
Ok(())
|
||||
};
|
||||
|
||||
@@ -6,10 +6,14 @@ use crate::structs::Price;
|
||||
|
||||
#[derive(Debug, Default, Allocative)]
|
||||
pub struct CapitalizationState {
|
||||
pub realized_cap: Price,
|
||||
realized_cap: Price,
|
||||
}
|
||||
|
||||
impl CapitalizationState {
|
||||
pub fn realized_cap(&self) -> Price {
|
||||
self.realized_cap
|
||||
}
|
||||
|
||||
pub fn increment(&mut self, realized_cap: Price) {
|
||||
self.realized_cap += realized_cap;
|
||||
}
|
||||
|
||||
@@ -2,11 +2,19 @@ use crate::structs::Amount;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct InputState {
|
||||
pub count: f64,
|
||||
pub volume: Amount,
|
||||
count: f64,
|
||||
volume: Amount,
|
||||
}
|
||||
|
||||
impl InputState {
|
||||
pub fn count(&self) -> f64 {
|
||||
self.count
|
||||
}
|
||||
|
||||
pub fn volume(&self) -> Amount {
|
||||
self.volume
|
||||
}
|
||||
|
||||
pub fn iterate(&mut self, count: f64, volume: Amount) {
|
||||
self.count += count;
|
||||
self.volume += volume;
|
||||
|
||||
@@ -2,11 +2,19 @@ use crate::structs::Amount;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct OutputState {
|
||||
pub count: f64,
|
||||
pub volume: Amount,
|
||||
count: f64,
|
||||
volume: Amount,
|
||||
}
|
||||
|
||||
impl OutputState {
|
||||
// pub fn count(&self) -> f64 {
|
||||
// self.count
|
||||
// }
|
||||
|
||||
// pub fn volume(&self) -> Amount {
|
||||
// self.volume
|
||||
// }
|
||||
|
||||
pub fn iterate(&mut self, count: f64, volume: Amount) {
|
||||
self.count += count;
|
||||
self.volume += volume;
|
||||
|
||||
@@ -2,30 +2,106 @@ use crate::structs::{Amount, Price};
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub struct PricePaidState {
|
||||
pub pp_05p: Option<Price>,
|
||||
pub pp_10p: Option<Price>,
|
||||
pub pp_15p: Option<Price>,
|
||||
pub pp_20p: Option<Price>,
|
||||
pub pp_25p: Option<Price>,
|
||||
pub pp_30p: Option<Price>,
|
||||
pub pp_35p: Option<Price>,
|
||||
pub pp_40p: Option<Price>,
|
||||
pub pp_45p: Option<Price>,
|
||||
pub pp_median: Option<Price>,
|
||||
pub pp_55p: Option<Price>,
|
||||
pub pp_60p: Option<Price>,
|
||||
pub pp_65p: Option<Price>,
|
||||
pub pp_70p: Option<Price>,
|
||||
pub pp_75p: Option<Price>,
|
||||
pub pp_80p: Option<Price>,
|
||||
pub pp_85p: Option<Price>,
|
||||
pub pp_90p: Option<Price>,
|
||||
pub pp_95p: Option<Price>,
|
||||
pp_05p: Option<Price>,
|
||||
pp_10p: Option<Price>,
|
||||
pp_15p: Option<Price>,
|
||||
pp_20p: Option<Price>,
|
||||
pp_25p: Option<Price>,
|
||||
pp_30p: Option<Price>,
|
||||
pp_35p: Option<Price>,
|
||||
pp_40p: Option<Price>,
|
||||
pp_45p: Option<Price>,
|
||||
pp_median: Option<Price>,
|
||||
pp_55p: Option<Price>,
|
||||
pp_60p: Option<Price>,
|
||||
pp_65p: Option<Price>,
|
||||
pp_70p: Option<Price>,
|
||||
pp_75p: Option<Price>,
|
||||
pp_80p: Option<Price>,
|
||||
pp_85p: Option<Price>,
|
||||
pp_90p: Option<Price>,
|
||||
pp_95p: Option<Price>,
|
||||
|
||||
pub processed_amount: Amount,
|
||||
processed_amount: Amount,
|
||||
}
|
||||
|
||||
impl PricePaidState {
|
||||
pub fn pp_05p(&self) -> Option<Price> {
|
||||
self.pp_05p
|
||||
}
|
||||
|
||||
pub fn pp_10p(&self) -> Option<Price> {
|
||||
self.pp_10p
|
||||
}
|
||||
|
||||
pub fn pp_15p(&self) -> Option<Price> {
|
||||
self.pp_15p
|
||||
}
|
||||
|
||||
pub fn pp_20p(&self) -> Option<Price> {
|
||||
self.pp_20p
|
||||
}
|
||||
|
||||
pub fn pp_25p(&self) -> Option<Price> {
|
||||
self.pp_25p
|
||||
}
|
||||
|
||||
pub fn pp_30p(&self) -> Option<Price> {
|
||||
self.pp_30p
|
||||
}
|
||||
|
||||
pub fn pp_35p(&self) -> Option<Price> {
|
||||
self.pp_35p
|
||||
}
|
||||
|
||||
pub fn pp_40p(&self) -> Option<Price> {
|
||||
self.pp_40p
|
||||
}
|
||||
|
||||
pub fn pp_45p(&self) -> Option<Price> {
|
||||
self.pp_45p
|
||||
}
|
||||
|
||||
pub fn pp_median(&self) -> Option<Price> {
|
||||
self.pp_median
|
||||
}
|
||||
|
||||
pub fn pp_55p(&self) -> Option<Price> {
|
||||
self.pp_55p
|
||||
}
|
||||
|
||||
pub fn pp_60p(&self) -> Option<Price> {
|
||||
self.pp_60p
|
||||
}
|
||||
|
||||
pub fn pp_65p(&self) -> Option<Price> {
|
||||
self.pp_65p
|
||||
}
|
||||
|
||||
pub fn pp_70p(&self) -> Option<Price> {
|
||||
self.pp_70p
|
||||
}
|
||||
|
||||
pub fn pp_75p(&self) -> Option<Price> {
|
||||
self.pp_75p
|
||||
}
|
||||
|
||||
pub fn pp_80p(&self) -> Option<Price> {
|
||||
self.pp_80p
|
||||
}
|
||||
|
||||
pub fn pp_85p(&self) -> Option<Price> {
|
||||
self.pp_85p
|
||||
}
|
||||
|
||||
pub fn pp_90p(&self) -> Option<Price> {
|
||||
self.pp_90p
|
||||
}
|
||||
|
||||
pub fn pp_95p(&self) -> Option<Price> {
|
||||
self.pp_95p
|
||||
}
|
||||
|
||||
pub fn iterate(&mut self, price: Price, amount: Amount, supply: Amount) {
|
||||
let PricePaidState {
|
||||
processed_amount: processed_supply,
|
||||
|
||||
@@ -2,23 +2,53 @@ use crate::structs::Price;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct RealizedState {
|
||||
pub realized_profit: Price,
|
||||
pub realized_loss: Price,
|
||||
pub value_created: Price,
|
||||
pub value_destroyed: Price,
|
||||
realized_profit: Price,
|
||||
realized_loss: Price,
|
||||
value_created: Price,
|
||||
adjusted_value_created: Price,
|
||||
value_destroyed: Price,
|
||||
adjusted_value_destroyed: Price,
|
||||
}
|
||||
|
||||
impl RealizedState {
|
||||
pub fn realized_profit(&self) -> Price {
|
||||
self.realized_profit
|
||||
}
|
||||
|
||||
pub fn realized_loss(&self) -> Price {
|
||||
self.realized_loss
|
||||
}
|
||||
|
||||
pub fn value_created(&self) -> Price {
|
||||
self.value_created
|
||||
}
|
||||
|
||||
pub fn adjusted_value_created(&self) -> Price {
|
||||
self.adjusted_value_created
|
||||
}
|
||||
|
||||
pub fn value_destroyed(&self) -> Price {
|
||||
self.value_destroyed
|
||||
}
|
||||
|
||||
pub fn adjusted_value_destroyed(&self) -> Price {
|
||||
self.adjusted_value_destroyed
|
||||
}
|
||||
|
||||
pub fn iterate(
|
||||
&mut self,
|
||||
realized_profit: Price,
|
||||
realized_loss: Price,
|
||||
value_created: Price,
|
||||
adjusted_value_created: Price,
|
||||
value_destroyed: Price,
|
||||
adjusted_value_destroyed: Price,
|
||||
) {
|
||||
self.realized_profit += realized_profit;
|
||||
self.realized_loss += realized_loss;
|
||||
self.value_created += value_created;
|
||||
self.adjusted_value_created += adjusted_value_created;
|
||||
self.value_destroyed += value_destroyed;
|
||||
self.adjusted_value_destroyed += adjusted_value_destroyed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,14 @@ use crate::structs::Amount;
|
||||
|
||||
#[derive(Debug, Default, Allocative)]
|
||||
pub struct SupplyState {
|
||||
pub supply: Amount,
|
||||
supply: Amount,
|
||||
}
|
||||
|
||||
impl SupplyState {
|
||||
pub fn supply(&self) -> Amount {
|
||||
self.supply
|
||||
}
|
||||
|
||||
pub fn increment(&mut self, amount: Amount) {
|
||||
self.supply += amount;
|
||||
}
|
||||
|
||||
@@ -4,12 +4,24 @@ use crate::structs::{Amount, Price};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct UnrealizedState {
|
||||
pub supply_in_profit: Amount,
|
||||
pub unrealized_profit: Price,
|
||||
pub unrealized_loss: Price,
|
||||
supply_in_profit: Amount,
|
||||
unrealized_profit: Price,
|
||||
unrealized_loss: Price,
|
||||
}
|
||||
|
||||
impl UnrealizedState {
|
||||
pub fn supply_in_profit(&self) -> Amount {
|
||||
self.supply_in_profit
|
||||
}
|
||||
|
||||
pub fn unrealized_profit(&self) -> Price {
|
||||
self.unrealized_profit
|
||||
}
|
||||
|
||||
pub fn unrealized_loss(&self) -> Price {
|
||||
self.unrealized_loss
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn iterate(&mut self, price_then: Price, price_now: Price, amount: Amount) {
|
||||
match price_then.cmp(&price_now) {
|
||||
|
||||
@@ -5,10 +5,14 @@ use color_eyre::eyre::eyre;
|
||||
|
||||
#[derive(Debug, Default, Allocative)]
|
||||
pub struct UTXOState {
|
||||
pub count: f64,
|
||||
count: f64,
|
||||
}
|
||||
|
||||
impl UTXOState {
|
||||
pub fn count(&self) -> f64 {
|
||||
self.count
|
||||
}
|
||||
|
||||
pub fn increment(&mut self, utxo_count: f64) {
|
||||
self.count += utxo_count;
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ impl UTXOCohortDurableStates {
|
||||
.replace(UnrealizedState::default());
|
||||
}
|
||||
|
||||
let supply = self.durable_states.supply_state.supply;
|
||||
let supply = self.durable_states.supply_state.supply();
|
||||
|
||||
let one_shot_states_ref = &mut one_shot_states;
|
||||
|
||||
|
||||
@@ -5,9 +5,7 @@ use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
states::DateDataVec,
|
||||
structs::{Amount, BlockData, Price, SentData},
|
||||
utils::difference_in_days_between_timestamps,
|
||||
Date,
|
||||
structs::{Amount, BlockData, Price, SentData, Timestamp},
|
||||
};
|
||||
|
||||
use super::{SplitByUTXOCohort, UTXOCohortDurableStates, UTXOCohortsOneShotStates};
|
||||
@@ -32,7 +30,7 @@ impl UTXOCohortsDurableStates {
|
||||
return;
|
||||
}
|
||||
|
||||
let increment_days_old = difference_in_days_between_timestamps(
|
||||
let increment_days_old = Timestamp::difference_in_days_between(
|
||||
block_data.timestamp,
|
||||
last_block_data.timestamp,
|
||||
);
|
||||
@@ -65,18 +63,18 @@ impl UTXOCohortsDurableStates {
|
||||
}
|
||||
|
||||
if block_data.height == last_block_data.height {
|
||||
let year = Date::from_timestamp(block_data.timestamp).year() as u32;
|
||||
let year = block_data.timestamp.to_year();
|
||||
|
||||
self.initial_filtered_apply(&0, &year, |state| {
|
||||
state.increment(amount, utxo_count, price).unwrap();
|
||||
})
|
||||
} else {
|
||||
let increment_days_old = difference_in_days_between_timestamps(
|
||||
let increment_days_old = Timestamp::difference_in_days_between(
|
||||
block_data.timestamp,
|
||||
last_block_data.timestamp,
|
||||
);
|
||||
|
||||
let decrement_days_old = difference_in_days_between_timestamps(
|
||||
let decrement_days_old = Timestamp::difference_in_days_between(
|
||||
block_data.timestamp,
|
||||
previous_last_block_data
|
||||
.unwrap_or_else(|| {
|
||||
@@ -117,12 +115,12 @@ impl UTXOCohortsDurableStates {
|
||||
return;
|
||||
}
|
||||
|
||||
let days_old = difference_in_days_between_timestamps(
|
||||
let days_old = Timestamp::difference_in_days_between(
|
||||
block_data.timestamp,
|
||||
previous_last_block_data.timestamp,
|
||||
);
|
||||
|
||||
let year = Date::from_timestamp(block_data.timestamp).year() as u32;
|
||||
let year = block_data.timestamp.to_year();
|
||||
|
||||
self.initial_filtered_apply(&days_old, &year, |state| {
|
||||
state
|
||||
|
||||
@@ -5,8 +5,7 @@ use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{
|
||||
states::{DateDataVec, InputState, RealizedState},
|
||||
structs::{BlockPath, Price, SentData},
|
||||
utils::difference_in_days_between_timestamps,
|
||||
structs::{BlockPath, Price, SentData, Timestamp},
|
||||
};
|
||||
|
||||
use super::SplitByUTXOCohort;
|
||||
@@ -26,6 +25,7 @@ impl UTXOCohortsSentStates {
|
||||
date_data_vec: &DateDataVec,
|
||||
block_path_to_sent_data: &BTreeMap<BlockPath, SentData>,
|
||||
current_price: Price,
|
||||
current_timestamp: Timestamp,
|
||||
) {
|
||||
if let Some(last_block_data) = date_data_vec.last_block() {
|
||||
block_path_to_sent_data
|
||||
@@ -37,11 +37,12 @@ impl UTXOCohortsSentStates {
|
||||
|
||||
let block_data = date_data.get_block_data(block_path).unwrap();
|
||||
|
||||
let days_old = difference_in_days_between_timestamps(
|
||||
let days_old = Timestamp::difference_in_days_between(
|
||||
block_data.timestamp,
|
||||
last_block_data.timestamp,
|
||||
);
|
||||
|
||||
let previous_timestamp = block_data.timestamp;
|
||||
let previous_price = block_data.price;
|
||||
|
||||
let amount_sent = sent_data.volume;
|
||||
@@ -52,18 +53,34 @@ impl UTXOCohortsSentStates {
|
||||
let previous_value = previous_price * amount_sent;
|
||||
let current_value = current_price * amount_sent;
|
||||
|
||||
state.realized.value_destroyed += previous_value;
|
||||
state.realized.value_created += current_value;
|
||||
let mut realized_profit = Price::ZERO;
|
||||
let mut realized_loss = Price::ZERO;
|
||||
let value_created = current_value;
|
||||
let mut adjusted_value_created = Price::ZERO;
|
||||
let value_destroyed = previous_value;
|
||||
let mut adjusted_value_destroyed = Price::ZERO;
|
||||
|
||||
match previous_value.cmp(¤t_value) {
|
||||
Ordering::Less => {
|
||||
state.realized.realized_profit += current_value - previous_value;
|
||||
}
|
||||
Ordering::Less => realized_profit = current_value - previous_value,
|
||||
Ordering::Greater => {
|
||||
state.realized.realized_loss += previous_value - current_value;
|
||||
realized_loss = previous_value - current_value;
|
||||
}
|
||||
Ordering::Equal => {}
|
||||
}
|
||||
|
||||
if previous_timestamp.older_by_1h_plus_than(current_timestamp) {
|
||||
adjusted_value_created = value_created;
|
||||
adjusted_value_destroyed = value_destroyed;
|
||||
}
|
||||
|
||||
state.realized.iterate(
|
||||
realized_profit,
|
||||
realized_loss,
|
||||
value_created,
|
||||
adjusted_value_created,
|
||||
value_destroyed,
|
||||
adjusted_value_destroyed,
|
||||
);
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use super::{AddressData, Amount, Price};
|
||||
use super::{AddressData, Amount, Price, Timestamp};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct AddressRealizedData {
|
||||
@@ -8,7 +8,9 @@ pub struct AddressRealizedData {
|
||||
pub profit: Price,
|
||||
pub loss: Price,
|
||||
pub value_created: Price,
|
||||
pub adjusted_value_created: Price,
|
||||
pub value_destroyed: Price,
|
||||
pub adjusted_value_destroyed: Price,
|
||||
pub utxos_created: u32,
|
||||
pub utxos_destroyed: u32,
|
||||
}
|
||||
@@ -23,7 +25,9 @@ impl AddressRealizedData {
|
||||
utxos_created: 0,
|
||||
utxos_destroyed: 0,
|
||||
value_created: Price::ZERO,
|
||||
adjusted_value_created: Price::ZERO,
|
||||
value_destroyed: Price::ZERO,
|
||||
adjusted_value_destroyed: Price::ZERO,
|
||||
initial_address_data: *initial_address_data,
|
||||
}
|
||||
}
|
||||
@@ -33,7 +37,14 @@ impl AddressRealizedData {
|
||||
self.utxos_created += 1;
|
||||
}
|
||||
|
||||
pub fn send(&mut self, amount: Amount, current_price: Price, previous_price: Price) {
|
||||
pub fn send(
|
||||
&mut self,
|
||||
amount: Amount,
|
||||
current_price: Price,
|
||||
previous_price: Price,
|
||||
current_timestamp: Timestamp,
|
||||
previous_timestamp: Timestamp,
|
||||
) {
|
||||
self.sent += amount;
|
||||
|
||||
self.utxos_destroyed += 1;
|
||||
@@ -44,6 +55,11 @@ impl AddressRealizedData {
|
||||
self.value_created += current_value;
|
||||
self.value_destroyed += previous_value;
|
||||
|
||||
if previous_timestamp.older_by_1h_plus_than(current_timestamp) {
|
||||
self.adjusted_value_created += current_value;
|
||||
self.adjusted_value_destroyed += previous_value;
|
||||
}
|
||||
|
||||
if current_value >= previous_value {
|
||||
self.profit += current_value - previous_value;
|
||||
} else {
|
||||
|
||||
@@ -36,6 +36,7 @@ direct_repr!(Amount);
|
||||
|
||||
impl Amount {
|
||||
pub const ZERO: Self = Self(BitcoinAmount::ZERO);
|
||||
pub const ONE_BTC_F32: f32 = 100_000_000.0;
|
||||
pub const ONE_BTC_F64: f64 = 100_000_000.0;
|
||||
|
||||
#[inline(always)]
|
||||
|
||||
@@ -299,6 +299,18 @@ where
|
||||
self.date
|
||||
.multi_insert_percentile(dates, date_map_and_percentiles, days);
|
||||
}
|
||||
|
||||
pub fn multi_insert_max(
|
||||
&mut self,
|
||||
heights: &[Height],
|
||||
dates: &[Date],
|
||||
source: &mut BiMap<Value>,
|
||||
) where
|
||||
Value: PartialOrd,
|
||||
{
|
||||
self.height.multi_insert_max(heights, &mut source.height);
|
||||
self.date.multi_insert_max(dates, &mut source.date);
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyBiMap {
|
||||
|
||||
@@ -2,19 +2,19 @@ use allocative::Allocative;
|
||||
use bincode::{Decode, Encode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{Amount, Height, Price};
|
||||
use super::{Amount, Height, Price, Timestamp};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
|
||||
pub struct BlockData {
|
||||
pub height: Height,
|
||||
pub price: Price,
|
||||
pub timestamp: u32,
|
||||
pub timestamp: Timestamp,
|
||||
pub amount: Amount,
|
||||
pub utxos: u32,
|
||||
}
|
||||
|
||||
impl BlockData {
|
||||
pub fn new(height: Height, price: Price, timestamp: u32) -> Self {
|
||||
pub fn new(height: Height, price: Price, timestamp: Timestamp) -> Self {
|
||||
Self {
|
||||
height,
|
||||
price,
|
||||
|
||||
@@ -33,20 +33,21 @@ pub struct Config {
|
||||
pub delay: Option<u64>,
|
||||
|
||||
/// Start a dry run, default: false, not saved
|
||||
#[arg(long, default_value_t = false)]
|
||||
pub dry_run: bool,
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
dry_run: Option<bool>,
|
||||
|
||||
/// Record ram usage, default: false, not saved
|
||||
#[arg(long, default_value_t = false)]
|
||||
pub record_ram_usage: bool,
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
record_ram_usage: Option<bool>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
const PATH: &'static str = "./config.toml";
|
||||
|
||||
pub fn import() -> color_eyre::Result<Self> {
|
||||
let mut config_saved = fs::read_to_string(Self::PATH)
|
||||
.map_or(Config::default(), |contents| {
|
||||
let mut config_saved =
|
||||
fs::read_to_string(Self::PATH).map_or(Config::default(), |contents| {
|
||||
dbg!(&contents);
|
||||
toml::from_str(&contents).unwrap_or_default()
|
||||
});
|
||||
|
||||
@@ -92,8 +93,8 @@ impl Config {
|
||||
log(&format!("rpcuser: {:?}", config.rpcuser));
|
||||
log(&format!("rpcpassword: {:?}", config.rpcpassword));
|
||||
log(&format!("delay: {:?}", config.delay));
|
||||
log(&format!("dry_run: {}", config.dry_run));
|
||||
log(&format!("record_ram_usage: {}", config.record_ram_usage));
|
||||
log(&format!("dry_run: {:?}", config.dry_run));
|
||||
log(&format!("record_ram_usage: {:?}", config.record_ram_usage));
|
||||
log("---");
|
||||
|
||||
Ok(config)
|
||||
@@ -124,4 +125,12 @@ impl Config {
|
||||
fn write(&self) -> std::io::Result<()> {
|
||||
fs::write(Self::PATH, toml::to_string(self).unwrap())
|
||||
}
|
||||
|
||||
pub fn dry_run(&self) -> bool {
|
||||
self.dry_run.is_some_and(|b| b)
|
||||
}
|
||||
|
||||
pub fn record_ram_usage(&self) -> bool {
|
||||
self.record_ram_usage.is_some_and(|b| b)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use bincode::{
|
||||
error::{DecodeError, EncodeError},
|
||||
BorrowDecode, Decode, Encode,
|
||||
};
|
||||
use chrono::{Datelike, Days, NaiveDate, TimeZone, Utc};
|
||||
use chrono::{Datelike, Days, NaiveDate};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -38,14 +38,6 @@ impl Date {
|
||||
Self(date)
|
||||
}
|
||||
|
||||
pub fn from_timestamp(timestamp: u32) -> Self {
|
||||
Self(
|
||||
Utc.timestamp_opt(i64::from(timestamp), 0)
|
||||
.unwrap()
|
||||
.date_naive(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn today() -> Self {
|
||||
Self(chrono::offset::Utc::now().date_naive())
|
||||
}
|
||||
|
||||
@@ -818,4 +818,36 @@ where
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
}
|
||||
|
||||
pub fn multi_insert_max(&mut self, keys: &[Key], source: &mut Self)
|
||||
where
|
||||
Value: Default + PartialOrd,
|
||||
{
|
||||
let mut max = None;
|
||||
|
||||
keys.iter().for_each(|key| {
|
||||
let previous_max = max.unwrap_or_else(|| {
|
||||
key.checked_sub(1)
|
||||
.and_then(|previous_max_key| self.get(&previous_max_key))
|
||||
.unwrap_or_default()
|
||||
});
|
||||
|
||||
let last_value = source.get_or_import(key).unwrap_or_else(|| {
|
||||
dbg!(key);
|
||||
panic!()
|
||||
});
|
||||
|
||||
if max.is_none() || last_value > previous_max {
|
||||
max.replace(last_value);
|
||||
}
|
||||
|
||||
self.insert(
|
||||
*key,
|
||||
max.unwrap_or_else(|| {
|
||||
dbg!(previous_max, last_value, max);
|
||||
panic!();
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use serde::{de::DeserializeOwned, Serialize};
|
||||
|
||||
use crate::datasets::OHLC;
|
||||
|
||||
use super::{Date, Height};
|
||||
use super::{Date, Height, Timestamp};
|
||||
|
||||
pub trait MapValue:
|
||||
Clone
|
||||
@@ -32,3 +32,4 @@ impl MapValue for f64 {}
|
||||
impl MapValue for Date {}
|
||||
impl MapValue for OHLC {}
|
||||
impl MapValue for Height {}
|
||||
impl MapValue for Timestamp {}
|
||||
|
||||
@@ -29,6 +29,7 @@ mod price;
|
||||
mod sent_data;
|
||||
mod serialized_btreemap;
|
||||
mod serialized_vec;
|
||||
mod timestamp;
|
||||
mod tx_data;
|
||||
mod txout_index;
|
||||
|
||||
@@ -63,5 +64,6 @@ pub use price::*;
|
||||
pub use sent_data::*;
|
||||
pub use serialized_btreemap::*;
|
||||
pub use serialized_vec::*;
|
||||
pub use timestamp::*;
|
||||
pub use tx_data::*;
|
||||
pub use txout_index::*;
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
use std::ops::Sub;
|
||||
|
||||
use allocative::Allocative;
|
||||
use bincode::{Decode, Encode};
|
||||
use chrono::{Datelike, NaiveDateTime, NaiveTime, TimeZone, Timelike, Utc};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::utils::{ONE_DAY_IN_S, ONE_HOUR_IN_S};
|
||||
|
||||
use super::Date;
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Default,
|
||||
Clone,
|
||||
Copy,
|
||||
Allocative,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
Deref,
|
||||
DerefMut,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
Encode,
|
||||
Decode,
|
||||
)]
|
||||
pub struct Timestamp(u32);
|
||||
|
||||
impl Timestamp {
|
||||
pub const ZERO: Self = Self(0);
|
||||
|
||||
pub fn wrap(timestamp: u32) -> Self {
|
||||
Self(timestamp)
|
||||
}
|
||||
|
||||
pub fn to_date(self) -> Date {
|
||||
Date::wrap(
|
||||
Utc.timestamp_opt(i64::from(self.0), 0)
|
||||
.unwrap()
|
||||
.date_naive(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn to_year(self) -> u32 {
|
||||
self.to_date().year() as u32
|
||||
}
|
||||
|
||||
pub fn to_floored_seconds(self) -> Self {
|
||||
let date_time = Utc.timestamp_opt(i64::from(self.0), 0).unwrap();
|
||||
|
||||
Self::wrap(
|
||||
NaiveDateTime::new(
|
||||
date_time.date_naive(),
|
||||
NaiveTime::from_hms_opt(date_time.hour(), date_time.minute(), 0).unwrap(),
|
||||
)
|
||||
.and_utc()
|
||||
.timestamp() as u32,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn difference_in_days_between(older: Self, younger: Self) -> u32 {
|
||||
if younger <= older {
|
||||
0
|
||||
} else {
|
||||
*(younger - older) / ONE_DAY_IN_S as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub fn older_by_1h_plus_than(&self, younger: Self) -> bool {
|
||||
younger.checked_sub(**self).unwrap_or_default() > ONE_HOUR_IN_S as u32
|
||||
}
|
||||
}
|
||||
|
||||
impl Sub for Timestamp {
|
||||
type Output = Self;
|
||||
|
||||
fn sub(self, rhs: Self) -> Self::Output {
|
||||
Self::wrap(self.0 - rhs.0)
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,6 @@ use std::time::Instant;
|
||||
|
||||
use crate::utils::log;
|
||||
|
||||
use super::ONE_DAY_IN_S;
|
||||
|
||||
pub fn time<F, T>(name: &str, function: F) -> T
|
||||
where
|
||||
F: FnOnce() -> T,
|
||||
@@ -16,11 +14,3 @@ where
|
||||
|
||||
returned
|
||||
}
|
||||
|
||||
pub fn difference_in_days_between_timestamps(older: u32, younger: u32) -> u32 {
|
||||
if younger <= older {
|
||||
0
|
||||
} else {
|
||||
(younger - older) / ONE_DAY_IN_S as u32
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+1
-3
@@ -450,9 +450,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "biter"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd2ea6b4c4046feaaf73f3dff1cc648328af021d50be95946e0e7a807c985b47"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"bitcoincore-rpc",
|
||||
|
||||
Vendored
+3
-3
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user