mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-08 06:01:57 -07:00
nym21
+8
-29
@@ -1,44 +1,19 @@
|
||||
# Mac OS
|
||||
.DS_Store
|
||||
|
||||
# To do
|
||||
/charts
|
||||
TODO.md
|
||||
|
||||
# Builds
|
||||
dist
|
||||
target
|
||||
|
||||
# I/O
|
||||
in
|
||||
out
|
||||
.log
|
||||
/datasets
|
||||
/datasets2
|
||||
/price
|
||||
*..*
|
||||
/txout_*
|
||||
/db
|
||||
|
||||
# Sync
|
||||
.stfolder
|
||||
|
||||
# Copies
|
||||
*\ copy*
|
||||
|
||||
# Ignored
|
||||
ignore
|
||||
|
||||
# Scripts
|
||||
/start-node.sh
|
||||
/_*
|
||||
|
||||
# Editors
|
||||
.vscode
|
||||
.zed
|
||||
|
||||
# Configs
|
||||
config.toml
|
||||
|
||||
# Flamegraph
|
||||
flamegraph/
|
||||
flamegraph.svg
|
||||
@@ -53,7 +28,11 @@ snapshots*/
|
||||
docker/kibo
|
||||
|
||||
# Types
|
||||
website/scripts/types/paths.d.ts
|
||||
paths.d.ts
|
||||
vecid-to-indexes.d.ts
|
||||
|
||||
# Misc
|
||||
OPENSATS.md
|
||||
# Outputs
|
||||
_outputs
|
||||
|
||||
# Python
|
||||
.ropeproject
|
||||
|
||||
+62
-31
@@ -1,13 +1,44 @@
|
||||
# Changelog
|
||||
|
||||
<!--
|
||||
## v. 0.X.Y | WIP
|
||||
|
||||
# v0.X.Y | WIP
|
||||
|
||||
-->
|
||||
|
||||
## v. 0.5.0 | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
# v0.6.0 | WIP | A new beginning
|
||||
|
||||
|
||||
## Global
|
||||
|
||||
- Completely redesign the back-end
|
||||
|
||||
- Merged parser and server crates into a single project (and thus executable), so now both will run at the same time with a single `cargo run -r` [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
|
||||
- Added `--no-server` and `--no-parser` to disable each if needed
|
||||
- Improved executable parameters
|
||||
- Started using `log` and `env_logger` crates instead of custom code [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
|
||||
- Improved logs
|
||||
- Fixed input being unfocused right after being focused in Brave browser [#9a9ae61](https://github.com/kibo-money/kibo/commit/9a9ae614d07b54c08b7e9c0e2aefe3b52fdb93c5)
|
||||
|
||||
- Reworked server's API code [#6ab0f46]( https://github.com/kibo-money/kibo/commit/6ab0f463119a902a1b7ca9691b54f61543bb8f2f)
|
||||
- New route format: `/api/date-to-realized-price` is now `/api/realized-price?kind=date`
|
||||
- Added status and timing to logs
|
||||
- Updated website packages
|
||||
- Added API support for datasets by timestamp (by merging any dataset by height with the height to timestamp dataset and so it still uses heights as chunk ids) [#ca00f3f](https://github.com/kibo-money/kibo/commit/ca00f3f71526f0c5c16021024fec7e5c6e47221c)
|
||||
- `/api/realized-price?kind=t`
|
||||
- `/api/realized-price?kind=timestamp&chunk=860000`
|
||||
- Created separate crate for indexing called `bindex`
|
||||
- Created a crate a storage engine specialized in storing datasets that have indexes as keys and thus can be represented by an array/vec called `storable-vec`
|
||||
- Removed the need for the `-txindex=1` parameter when starting your Bitcoin Core node as kibō has its own indexes now
|
||||
|
||||
## Git
|
||||
|
||||
Added git tags for each version though Markdown won't display formatted on Github so left the default text
|
||||
|
||||
## Deprecated
|
||||
|
||||
Moved Sanakirja database wrapper to its own crate (`snkrj`) and added a robust auto defragmentation to improve disk usage without the need for user's intervention.
|
||||
Since it's not used anymore it will moved out of the repository relatively soon.
|
||||
|
||||
# [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
|
||||
|
||||
|
||||
## Datasets
|
||||
|
||||
@@ -72,15 +103,15 @@
|
||||
|
||||
- Moved back to this repo
|
||||
|
||||
## v. 0.4.0 | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
# [v0.4.0](https://github.com/kibo-money/kibo/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
|
||||
|
||||
|
||||
|
||||
### Brand
|
||||
## Brand
|
||||
|
||||
- **Satonomics** is now **kibō** 🎉
|
||||
|
||||
### Website
|
||||
## Website
|
||||
|
||||
- Complete redesign of the website
|
||||
- Rewrote the whole application and removed `node`/`npm`/`pnpm` dependencies in favor for pure `HTML`/`CSS`/`Javascript`
|
||||
@@ -88,7 +119,7 @@
|
||||
- Added Trading View attribution link to the settings frame and file in the lightweight charts folder
|
||||
- Many other changes
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
|
||||
- Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage
|
||||
@@ -103,17 +134,17 @@
|
||||
- Various first run fixes
|
||||
- Added to `-h` which arguments are saved, which is all of them at the time of writing
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Updated the code to support compressed binaries
|
||||
- Added serving of the website
|
||||
- Improved `Cache-Control` behavior
|
||||
|
||||
## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
|
||||
# [v0.3.0](https://github.com/kibo-money/kibo/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
|
||||
|
||||
|
||||
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Global
|
||||
- Improved self-hosting by:
|
||||
@@ -156,7 +187,7 @@
|
||||
- Price
|
||||
- Improved error message when price cannot be found
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added chart scroll button for nice animations à la Wicked
|
||||
@@ -182,17 +213,17 @@
|
||||
- Settings
|
||||
- Removed the horizontal scroll bar which was unintended
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Run file
|
||||
- Only run with a watcher if `cargo watch` is available
|
||||
- Removed id_to_path file in favor for only `paths.d.ts` in `app/src/types`
|
||||
|
||||
## v. 0.2.0 | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
|
||||
# [v0.2.0](https://github.com/kibo-money/kibo/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
|
||||
|
||||
|
||||
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added the height version of all datasets and many optimizations to make them usable but only available on desktop and tablets for now
|
||||
@@ -220,24 +251,24 @@
|
||||
- Hopefully made scrollbars a little more subtle on WIndows and Linux, can't test
|
||||
- Generale style updates
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Fixed ulimit only being run in Mac OS instead of whenever the program is detected
|
||||
|
||||
## v. 0.1.1 | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
|
||||
# [v0.1.1](https://github.com/kibo-money/kibo/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
|
||||
|
||||
|
||||
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Fixed overflow in `Price` struct which caused many Realized Caps and Realized Prices to have completely bogus data
|
||||
- Fixed Realized Cap computation which was using rounded prices instead normal ones
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Added the chunk, date and time of the request to the terminal logs
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- Chart
|
||||
- Added double click option on a legend to toggle the visibility of all other series
|
||||
@@ -270,14 +301,14 @@
|
||||
- Misc
|
||||
- Removed tracker even though it was a very privacy friendly as it appeared to not be working properly
|
||||
|
||||
### Price
|
||||
## Price
|
||||
|
||||
- Deleted old price datasets and their backups
|
||||
|
||||
## v. 0.1.0 | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
|
||||
# [v0.1.0](https://github.com/kibo-money/kibo/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
|
||||
|
||||
|
||||
|
||||
|
||||
## v. 0.0.X | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
|
||||
# v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# Guidelines
|
||||
|
||||
## Parser
|
||||
|
||||
- Avoid floats as much as possible
|
||||
- Use structs like `WAmount` and `Price` for calculations
|
||||
- **Only** use `WAmount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
|
||||
- No `Arc`, `Rc`, `Mutex` even from third party libraries, they're slower
|
||||
Generated
+1316
-1050
File diff suppressed because it is too large
Load Diff
+31
@@ -0,0 +1,31 @@
|
||||
[workspace]
|
||||
members = ["crates/*"]
|
||||
resolver = "2"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.0.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
bitcoin = { version = "0.32.5", features = ["serde"] }
|
||||
brk_computer = { version = "0", path = "crates/brk_computer" }
|
||||
brk_fetcher = { version = "0", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0", path = "crates/brk_indexer" }
|
||||
brk_parser = { version = "0", path = "crates/brk_parser" }
|
||||
brk_logger = { version = "0", path = "crates/brk_logger" }
|
||||
brk_server = { version = "0", path = "crates/brk_server" }
|
||||
byteview = "0.5.4"
|
||||
color-eyre = "0.6.3"
|
||||
derive_deref = "1.1.1"
|
||||
fjall = "2.6.5"
|
||||
hodor = { version = "0", path = "crates/hodor" }
|
||||
jiff = "0.2.1"
|
||||
log = { version = "0.4.26" }
|
||||
minreq = { version = "2.13.2", features = ["https", "serde_json"] }
|
||||
rayon = "1.10.0"
|
||||
serde = { version = "1.0.218", features = ["derive"] }
|
||||
serde_bytes = "0.11.15"
|
||||
serde_json = { version = "1.0.139", features = ["float_roundtrip"] }
|
||||
storable_vec = { version = "0", path = "crates/storable_vec", features = [
|
||||
"json",
|
||||
] }
|
||||
zerocopy = { version = "0.8.20", features = ["derive"] }
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 kibō
|
||||
Copyright (c) 2025 kibō.money
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<a href="https://kibo.money" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/kibo-money/kibo/main/assets/logo-long-text-dark.svg">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/kibo-money/kibo/main/assets/logo-long-text-light.svg">
|
||||
<img alt="kibō" src="https://raw.githubusercontent.com/kibo-money/kibo/main/assets/logo-long-text-light.svg" width="210" height="auto">
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/kibo-money/kibo/main/_assets/logo-long-text-dark.svg">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/kibo-money/kibo/main/_assets/logo-long-text-light.svg">
|
||||
<img alt="kibō" src="https://raw.githubusercontent.com/kibo-money/kibo/main/_assets/logo-long-text-light.svg" width="210" height="auto">
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
@@ -63,17 +63,18 @@ Please open an issue if you want to add another instance
|
||||
### Requirements
|
||||
|
||||
- At least 16 GB of RAM
|
||||
- 1 TB of free space (will use 70% of that without defragmentation and 40% after)
|
||||
- A running instance of bitcoin-core with:
|
||||
- `-txindex=1`
|
||||
- `-blocksxor=0`
|
||||
- RPC credentials
|
||||
- Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly -txindex=1 -blocksxor=0`
|
||||
- A disk with 1 TB of free space (will use between 40% to 80% depending on several things)
|
||||
- Recommended: Rated at 3 GB/s (Thunderbolt 4 speed)
|
||||
- A running instance of bitcoin-core
|
||||
- Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly`
|
||||
- Git
|
||||
- Unix based operating system (Mac OS or Linux)
|
||||
- Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config`
|
||||
- Mac OS:
|
||||
- Disable Spotlight or exclude the `--kibodir` folder from it
|
||||
- Don't use Time Machine or exclude the `--kibodir` folder (especially needed for local snapshots)
|
||||
|
||||
### Manual
|
||||
|
||||
_Mac OS and Linux only, Windows is unsupported_
|
||||
### Build
|
||||
|
||||
First we need to install Rust (https://www.rust-lang.org/tools/install)
|
||||
|
||||
@@ -81,74 +82,33 @@ First we need to install Rust (https://www.rust-lang.org/tools/install)
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
```
|
||||
|
||||
If you already had Rust installed you could update it just in case
|
||||
If you already had Rust installed you could update it
|
||||
|
||||
```bash
|
||||
rustup update
|
||||
```
|
||||
|
||||
> If you're on Ubuntu you'll probably also need to install `open-ssl` with
|
||||
>
|
||||
> ```bash
|
||||
> sudo apt install libssl-dev pkg-config
|
||||
> ```
|
||||
|
||||
Optionally, you can also install `cargo-watch` for the server to automatically restart it on file change, which will be triggered by new code and new datasets from the parser (https://github.com/watchexec/cargo-watch?tab=readme-ov-file#install)
|
||||
|
||||
```bash
|
||||
cargo install cargo-watch --locked
|
||||
```
|
||||
|
||||
Then you need to choose a path where all files related to **kibō** will live
|
||||
Then you need to choose a path where the project will reside and then clone it
|
||||
|
||||
```bash
|
||||
cd ???
|
||||
```
|
||||
|
||||
We can now clone the repository
|
||||
|
||||
```bash
|
||||
git clone https://github.com/kibo-money/kibo.git
|
||||
cd kibo
|
||||
```
|
||||
|
||||
In a new terminal, go to the `parser`'s folder of the repository
|
||||
If it's your first time running kibo, it will need several information such as:
|
||||
|
||||
```bash
|
||||
cd ???/kibo/parser
|
||||
```
|
||||
- `--bitcoindir PATH`: path to bitcoin core data directory, `???/bitcoin`
|
||||
- `--kibodir PATH`: path to kibo outputs, if you have enough space on your main disk `~/.kibo` is fine
|
||||
|
||||
Now we can finally start by running the parser, you need to use the `./run.sh` script instead of `cargo run -r` as we need to set various system variables for the program to run smoothly
|
||||
Everything will be saved at `~/.kibo/config.toml`, which will allow you to simply run `cargo run -r` next time
|
||||
|
||||
For the first launch, the parser will need several information such as:
|
||||
|
||||
- `--datadir`: which is bitcoin data directory path, prefer `$HOME` to `~` as the latter might not work
|
||||
|
||||
Optionally you can also specify:
|
||||
|
||||
- `--rpccookiefile`: the path to the cookie file if not default
|
||||
- `--rpcuser`: the username of the RPC credentials to talk to the bitcoin server if set
|
||||
- `--rpcpassword`: the password of the RPC credentials if set
|
||||
- `--rpcconnect`: if the bitcoin core server's IP is different than `localhost`
|
||||
- `--rpcport`: if the port is different than `8332`
|
||||
|
||||
Everything will be saved in a `config.toml` file, which will allow you to simply run `./run.sh` next time
|
||||
If you need more options please run `cargo run -r --help` to see what parameters are available.
|
||||
|
||||
Here's an example
|
||||
|
||||
```bash
|
||||
./run.sh --datadir=$HOME/Developer/bitcoin
|
||||
```
|
||||
|
||||
In a **new** terminal, go to the `server`'s folder of the repository
|
||||
|
||||
```bash
|
||||
cd ???/kibo/server
|
||||
```
|
||||
|
||||
And start it also with the `run.sh` script instead of `cargo run -r`
|
||||
|
||||
```bash
|
||||
./run.sh
|
||||
cargo run -r -- --bitcoindir=~/Developer/bitcoin --kibodir=~/.kibo
|
||||
```
|
||||
|
||||
Then the easiest to let others access your server is to use `cloudflared` which will also cache requests. For more information go to: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/
|
||||
|
||||
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 1.0 KiB After Width: | Height: | Size: 1.0 KiB |
|
Before Width: | Height: | Size: 1.0 KiB After Width: | Height: | Size: 1.0 KiB |
|
Before Width: | Height: | Size: 1.0 KiB After Width: | Height: | Size: 1.0 KiB |
|
Before Width: | Height: | Size: 5.6 KiB After Width: | Height: | Size: 5.6 KiB |
|
Before Width: | Height: | Size: 5.6 KiB After Width: | Height: | Size: 5.6 KiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 4.4 KiB After Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 4.4 KiB After Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 1.8 MiB After Width: | Height: | Size: 1.8 MiB |
|
Before Width: | Height: | Size: 1.8 MiB After Width: | Height: | Size: 1.8 MiB |
|
Before Width: | Height: | Size: 496 KiB After Width: | Height: | Size: 496 KiB |
|
Before Width: | Height: | Size: 564 KiB After Width: | Height: | Size: 564 KiB |
|
Before Width: | Height: | Size: 592 KiB After Width: | Height: | Size: 592 KiB |
|
Before Width: | Height: | Size: 453 KiB After Width: | Height: | Size: 453 KiB |
|
Before Width: | Height: | Size: 526 KiB After Width: | Height: | Size: 526 KiB |
|
Before Width: | Height: | Size: 208 KiB After Width: | Height: | Size: 208 KiB |
|
Before Width: | Height: | Size: 386 KiB After Width: | Height: | Size: 386 KiB |
@@ -0,0 +1,67 @@
|
||||
use brk_parser::bitcoincore_rpc::Client;
|
||||
use log::info;
|
||||
use rlimit::{Resource, getrlimit, setrlimit};
|
||||
|
||||
mod io;
|
||||
mod parser;
|
||||
mod server;
|
||||
mod structs;
|
||||
mod utils;
|
||||
|
||||
use brk_parser::Datasets;
|
||||
use server::api::structs::Routes;
|
||||
use structs::{Config, Exit};
|
||||
use utils::init_log;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
init_log();
|
||||
|
||||
let (_, nofile_limit) = getrlimit(Resource::NOFILE).unwrap();
|
||||
setrlimit(Resource::NOFILE, 138_240, nofile_limit)?;
|
||||
|
||||
std::thread::Builder::new()
|
||||
.stack_size(getrlimit(Resource::STACK).unwrap().1 as usize)
|
||||
.spawn(|| -> color_eyre::Result<()> {
|
||||
let exit = Exit::new();
|
||||
|
||||
let config = Config::import()?;
|
||||
|
||||
info!("Starting...");
|
||||
|
||||
let rpc = Client::from(&config);
|
||||
|
||||
let routes = Routes::build(&Datasets::import(&config)?, &config);
|
||||
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
let run_parser = config.parser();
|
||||
let run_server = config.server();
|
||||
|
||||
let config_clone = config.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
if run_server {
|
||||
server::main(routes, config_clone).await.unwrap();
|
||||
} else {
|
||||
info!("Skipping server");
|
||||
}
|
||||
});
|
||||
|
||||
if run_parser {
|
||||
parser::main(&config, &rpc, &exit)?;
|
||||
} else {
|
||||
info!("Skipping parser");
|
||||
}
|
||||
|
||||
handle.await?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
})?
|
||||
.join()
|
||||
.unwrap()
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
use log::info;
|
||||
|
||||
use crate::{
|
||||
parser::{databases::Databases, datasets::Datasets, states::States},
|
||||
structs::{Config, Date, Exit, Height},
|
||||
utils::time,
|
||||
};
|
||||
|
||||
pub struct ExportedData<'a> {
|
||||
pub config: &'a Config,
|
||||
pub databases: Option<&'a mut Databases>,
|
||||
pub datasets: &'a mut Datasets,
|
||||
pub date: Date,
|
||||
pub defragment: bool,
|
||||
pub exit: Exit,
|
||||
pub height: Height,
|
||||
pub states: Option<&'a States>,
|
||||
}
|
||||
|
||||
pub fn export(
|
||||
ExportedData {
|
||||
config,
|
||||
databases,
|
||||
datasets,
|
||||
date,
|
||||
defragment,
|
||||
exit,
|
||||
height,
|
||||
states,
|
||||
}: ExportedData,
|
||||
) -> color_eyre::Result<()> {
|
||||
if exit.active() {
|
||||
info!("Exit in progress, skipping export");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
exit.block();
|
||||
|
||||
let text = if defragment {
|
||||
"Exporting and defragmenting..."
|
||||
} else {
|
||||
"Exporting..."
|
||||
};
|
||||
info!("{text}");
|
||||
|
||||
time("Finished export", || -> color_eyre::Result<()> {
|
||||
datasets.export(config, height)?;
|
||||
|
||||
if let Some(databases) = databases {
|
||||
databases.export(height, date, defragment)?;
|
||||
}
|
||||
|
||||
if let Some(states) = states {
|
||||
states.export(config)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
exit.unblock();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,70 +1,56 @@
|
||||
use std::{collections::BTreeSet, time::Instant};
|
||||
|
||||
use brk_parser::bitcoincore_rpc::Client;
|
||||
use chrono::Datelike;
|
||||
use export::ExportedData;
|
||||
use itertools::Itertools;
|
||||
|
||||
use log::info;
|
||||
use parse::ParseData;
|
||||
|
||||
use crate::{
|
||||
actions::{export, find_first_inserted_unsafe_height, parse},
|
||||
create_rpc,
|
||||
databases::Databases,
|
||||
datasets::{AllDatasets, ComputeData},
|
||||
io::OUTPUTS_FOLDER_PATH,
|
||||
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
|
||||
structs::{DateData, MapKey, Timestamp},
|
||||
utils::{generate_allocation_files, log, time},
|
||||
Config, Exit, Height,
|
||||
parser::{
|
||||
actions::{export, find_first_inserted_unsafe_height, parse},
|
||||
databases::Databases,
|
||||
datasets::{ComputeData, Datasets},
|
||||
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
|
||||
},
|
||||
structs::{Config, DateData, DisplayInstant, Exit, Height, MapKey, Timestamp},
|
||||
utils::{generate_allocation_files, time},
|
||||
};
|
||||
|
||||
pub fn iter_blocks(
|
||||
config: &mut Config,
|
||||
rpc: &biter::bitcoincore_rpc::Client,
|
||||
config: &Config,
|
||||
rpc: &Client,
|
||||
approx_block_count: usize,
|
||||
exit: Exit,
|
||||
databases: &mut Databases,
|
||||
datasets: &mut Datasets,
|
||||
) -> color_eyre::Result<()> {
|
||||
log("Starting...");
|
||||
let mut states = States::import(config).unwrap_or_default();
|
||||
|
||||
let mut datasets = AllDatasets::import(config)?;
|
||||
info!("Imported states");
|
||||
|
||||
log("Imported datasets");
|
||||
|
||||
let mut databases = Databases::import();
|
||||
|
||||
if config.first_defragment() {
|
||||
databases.defragment(&exit);
|
||||
config.disable_defragment();
|
||||
}
|
||||
|
||||
log("Imported databases");
|
||||
|
||||
let mut states = States::import().unwrap_or_default();
|
||||
|
||||
log("Imported states");
|
||||
|
||||
let first_unsafe_heights =
|
||||
find_first_inserted_unsafe_height(&mut states, &mut databases, &mut datasets);
|
||||
let first_unsafe_heights = find_first_inserted_unsafe_height(&mut states, databases, datasets, config);
|
||||
|
||||
let mut height = first_unsafe_heights.min();
|
||||
|
||||
log(&format!("Starting parsing at height: {height}"));
|
||||
info!("Starting parsing at height: {height}");
|
||||
|
||||
let mut next_block_opt = None;
|
||||
let mut blocks_loop_date = None;
|
||||
let mut next_date_opt;
|
||||
|
||||
let block_receiver = biter::new(
|
||||
config.datadir.as_ref().unwrap(),
|
||||
OUTPUTS_FOLDER_PATH,
|
||||
&config.path_bitcoindir(),
|
||||
Some(height.to_usize()),
|
||||
None,
|
||||
create_rpc(config).unwrap(),
|
||||
Client::from(config),
|
||||
);
|
||||
|
||||
let mut block_iter = block_receiver.iter();
|
||||
|
||||
'parsing: loop {
|
||||
let instant = Instant::now();
|
||||
|
||||
let mut processed_heights = BTreeSet::new();
|
||||
let mut processed_dates = BTreeSet::new();
|
||||
|
||||
@@ -75,15 +61,15 @@ pub fn iter_blocks(
|
||||
blocks_loop_date.take();
|
||||
}
|
||||
|
||||
let instant = Instant::now();
|
||||
|
||||
'blocks: loop {
|
||||
let current_block_opt = next_block_opt.take().or_else(|| block_iter.next());
|
||||
|
||||
next_block_opt = block_iter.next();
|
||||
|
||||
if let Some((_current_block_height, current_block, _current_block_hash)) =
|
||||
current_block_opt
|
||||
{
|
||||
let timestamp = Timestamp::wrap(current_block.header.time);
|
||||
if let Some((_current_block_height, current_block, _current_block_hash)) = current_block_opt {
|
||||
let timestamp = Timestamp::from(current_block.header.time);
|
||||
|
||||
let current_block_date = timestamp.to_date();
|
||||
let current_block_height: Height = height + blocks_loop_i;
|
||||
@@ -93,16 +79,12 @@ pub fn iter_blocks(
|
||||
panic!()
|
||||
}
|
||||
|
||||
let next_block_date = next_block_opt.as_ref().map(|(_, next_block, _)| {
|
||||
Timestamp::wrap(next_block.header.time).to_date()
|
||||
});
|
||||
next_date_opt = next_block_opt
|
||||
.as_ref()
|
||||
.map(|(_, next_block, _)| Timestamp::from(next_block.header.time).to_date());
|
||||
|
||||
// Always run for the first block of the loop
|
||||
if blocks_loop_date.is_none() {
|
||||
log(&format!(
|
||||
"Processing {current_block_date} (height: {height})..."
|
||||
));
|
||||
|
||||
blocks_loop_date.replace(current_block_date);
|
||||
|
||||
if states
|
||||
@@ -111,9 +93,7 @@ pub fn iter_blocks(
|
||||
.map(|date_data| *date_data.date < *current_block_date)
|
||||
.unwrap_or(true)
|
||||
{
|
||||
states
|
||||
.date_data_vec
|
||||
.push(DateData::new(current_block_date, vec![]));
|
||||
states.date_data_vec.push(DateData::new(current_block_date, vec![]));
|
||||
}
|
||||
|
||||
processed_dates.insert(current_block_date);
|
||||
@@ -125,17 +105,15 @@ pub fn iter_blocks(
|
||||
panic!("current block should always have the same date as the current blocks loop");
|
||||
}
|
||||
|
||||
let is_date_last_block = next_block_date
|
||||
let is_date_last_block = next_date_opt
|
||||
// Do NOT change `blocks_loop_date` to `current_block_date` !!!
|
||||
.map_or(true, |next_block_date| blocks_loop_date < next_block_date);
|
||||
|
||||
processed_heights.insert(current_block_height);
|
||||
|
||||
if first_unsafe_heights.inserted <= current_block_height {
|
||||
let compute_addresses = databases.check_if_needs_to_compute_addresses(
|
||||
current_block_height,
|
||||
blocks_loop_date,
|
||||
);
|
||||
let compute_addresses =
|
||||
databases.check_if_needs_to_compute_addresses(current_block_height, blocks_loop_date);
|
||||
|
||||
if states.address_cohorts_durable_states.is_none()
|
||||
&& (compute_addresses
|
||||
@@ -143,10 +121,9 @@ pub fn iter_blocks(
|
||||
.address
|
||||
.needs_durable_states(current_block_height, current_block_date))
|
||||
{
|
||||
states.address_cohorts_durable_states =
|
||||
Some(AddressCohortsDurableStates::init(
|
||||
&mut databases.address_index_to_address_data,
|
||||
));
|
||||
states.address_cohorts_durable_states = Some(AddressCohortsDurableStates::init(
|
||||
&mut databases.address_index_to_address_data,
|
||||
));
|
||||
}
|
||||
|
||||
if states.utxo_cohorts_durable_states.is_none()
|
||||
@@ -159,16 +136,17 @@ pub fn iter_blocks(
|
||||
}
|
||||
|
||||
parse(ParseData {
|
||||
rpc,
|
||||
block: current_block,
|
||||
block_index: blocks_loop_i,
|
||||
compute_addresses,
|
||||
databases: &mut databases,
|
||||
datasets: &mut datasets,
|
||||
config,
|
||||
databases,
|
||||
datasets,
|
||||
date: blocks_loop_date,
|
||||
first_date_height: height,
|
||||
height: current_block_height,
|
||||
is_date_last_block,
|
||||
rpc,
|
||||
states: &mut states,
|
||||
});
|
||||
}
|
||||
@@ -176,15 +154,16 @@ pub fn iter_blocks(
|
||||
blocks_loop_i += 1;
|
||||
|
||||
if is_date_last_block {
|
||||
info!(
|
||||
"Processed {current_block_date} ({height} - {current_block_height}) {}",
|
||||
instant.display()
|
||||
);
|
||||
|
||||
height += blocks_loop_i;
|
||||
|
||||
let is_check_point = next_block_date
|
||||
.as_ref()
|
||||
.map_or(true, |date| date.is_first_of_month());
|
||||
let is_check_point = next_date_opt.as_ref().map_or(true, |date| date.is_first_of_month());
|
||||
|
||||
let ran_for_at_least_a_minute = instant.elapsed().as_secs() >= 60;
|
||||
|
||||
if (is_check_point && ran_for_at_least_a_minute)
|
||||
if (is_check_point && instant.elapsed().as_secs() >= 1)
|
||||
|| height.is_close_to_end(approx_block_count)
|
||||
{
|
||||
break 'days;
|
||||
@@ -201,14 +180,9 @@ pub fn iter_blocks(
|
||||
// Don't remember why -1
|
||||
let last_height = height - 1_u32;
|
||||
|
||||
log(&format!(
|
||||
"Parsing group took {} seconds (last height: {last_height})\n",
|
||||
instant.elapsed().as_secs_f32(),
|
||||
));
|
||||
|
||||
if first_unsafe_heights.computed <= last_height {
|
||||
log("Computing datasets...");
|
||||
time("Computing datasets", || {
|
||||
info!("Computing datasets...");
|
||||
time("Computed datasets", || {
|
||||
let dates = processed_dates.into_iter().collect_vec();
|
||||
|
||||
let heights = processed_heights.into_iter().collect_vec();
|
||||
@@ -223,25 +197,31 @@ pub fn iter_blocks(
|
||||
if !config.dry_run() {
|
||||
let is_safe = height.is_safe(approx_block_count);
|
||||
|
||||
let defragment = is_safe
|
||||
&& next_date_opt.is_some_and(|date| {
|
||||
(date.year() >= 2020 && date.is_january() || date.year() >= 2022 && date.is_july())
|
||||
&& date.is_first_of_month()
|
||||
});
|
||||
|
||||
export(ExportedData {
|
||||
databases: is_safe.then_some(&mut databases),
|
||||
datasets: &mut datasets,
|
||||
config,
|
||||
databases: is_safe.then_some(databases),
|
||||
datasets,
|
||||
date: blocks_loop_date.unwrap(),
|
||||
defragment,
|
||||
height: last_height,
|
||||
states: is_safe.then_some(&states),
|
||||
exit: exit.clone(),
|
||||
})?;
|
||||
|
||||
if config.record_ram_usage() {
|
||||
time("Exporing allocation files", || {
|
||||
generate_allocation_files(&datasets, &databases, &states, last_height)
|
||||
time("Exporting allocation files", || {
|
||||
generate_allocation_files(datasets, databases, &states, last_height)
|
||||
})?;
|
||||
}
|
||||
} else {
|
||||
log("Skipping export");
|
||||
info!("Skipping export");
|
||||
}
|
||||
|
||||
println!();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1,9 +1,12 @@
|
||||
use log::info;
|
||||
|
||||
use crate::{
|
||||
databases::Databases,
|
||||
datasets::{AllDatasets, AnyDatasets},
|
||||
states::States,
|
||||
structs::Height,
|
||||
utils::log,
|
||||
parser::{
|
||||
databases::Databases,
|
||||
datasets::{AnyDatasets, Datasets},
|
||||
states::States,
|
||||
},
|
||||
structs::{Config, Height},
|
||||
};
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
@@ -21,7 +24,8 @@ impl Heights {
|
||||
pub fn find_first_inserted_unsafe_height(
|
||||
states: &mut States,
|
||||
databases: &mut Databases,
|
||||
datasets: &mut AllDatasets,
|
||||
datasets: &mut Datasets,
|
||||
config: &Config,
|
||||
) -> Heights {
|
||||
let min_initial_inserted_last_address_height = datasets
|
||||
.address
|
||||
@@ -51,7 +55,7 @@ pub fn find_first_inserted_unsafe_height(
|
||||
.map(|date_data| date_data.date)
|
||||
.and_then(|last_safe_date| {
|
||||
if !usable_databases {
|
||||
log("Unusable databases");
|
||||
info!("Unusable databases");
|
||||
|
||||
return None;
|
||||
}
|
||||
@@ -61,8 +65,8 @@ pub fn find_first_inserted_unsafe_height(
|
||||
let min_datasets_inserted_last_height = datasets_min_initial_states.inserted.last_height;
|
||||
let min_datasets_inserted_last_date = datasets_min_initial_states.inserted.last_date;
|
||||
|
||||
log(&format!("min_datasets_inserted_last_height: {:?}", min_datasets_inserted_last_height));
|
||||
log(&format!("min_datasets_inserted_last_date: {:?}", min_datasets_inserted_last_date));
|
||||
info!("min_datasets_inserted_last_height: {:?}", min_datasets_inserted_last_height);
|
||||
info!("min_datasets_inserted_last_date: {:?}", min_datasets_inserted_last_date);
|
||||
|
||||
let inserted_last_date_is_older_than_saved_state = min_datasets_inserted_last_date.map_or(true, |min_datasets_last_date| min_datasets_last_date < last_safe_date);
|
||||
|
||||
@@ -80,7 +84,7 @@ pub fn find_first_inserted_unsafe_height(
|
||||
let inserted_heights_and_dates_are_out_of_sync = min_datasets_inserted_last_height.map_or(true, |min_datasets_inserted_last_height| min_datasets_inserted_last_height < last_safe_height);
|
||||
|
||||
if inserted_heights_and_dates_are_out_of_sync {
|
||||
log(&format!("last_safe_height ({last_safe_height}) > min_datasets_height ({min_datasets_inserted_last_height:?})"));
|
||||
info!("last_safe_height ({last_safe_height}) > min_datasets_height ({min_datasets_inserted_last_height:?})");
|
||||
|
||||
None
|
||||
} else {
|
||||
@@ -108,18 +112,13 @@ pub fn find_first_inserted_unsafe_height(
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
log("Starting over...");
|
||||
info!("Starting over...");
|
||||
|
||||
let include_addresses = !usable_databases
|
||||
|| min_initial_inserted_last_address_date.is_none()
|
||||
|| min_initial_inserted_last_address_height.is_none();
|
||||
|
||||
// if include_addresses {
|
||||
// dbg!(include_addresses);
|
||||
// panic!("");
|
||||
// }
|
||||
|
||||
states.reset(include_addresses);
|
||||
states.reset(config, include_addresses);
|
||||
|
||||
databases.reset(include_addresses);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::{collections::BTreeMap, ops::ControlFlow, thread};
|
||||
|
||||
use biter::{
|
||||
use brk_parser::{
|
||||
bitcoin::{Block, Txid},
|
||||
bitcoincore_rpc::RpcApi,
|
||||
};
|
||||
@@ -9,17 +9,19 @@ use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
databases::{
|
||||
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex,
|
||||
Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount,
|
||||
},
|
||||
datasets::{AllDatasets, InsertData},
|
||||
states::{
|
||||
AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates,
|
||||
States, UTXOCohortsOneShotStates, UTXOCohortsSentStates,
|
||||
parser::{
|
||||
databases::{
|
||||
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex, Databases, TxidToTxData,
|
||||
TxoutIndexToAddressIndex, TxoutIndexToAmount,
|
||||
},
|
||||
datasets::{Datasets, InsertData},
|
||||
states::{
|
||||
AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates, States,
|
||||
UTXOCohortsOneShotStates, UTXOCohortsSentStates,
|
||||
},
|
||||
},
|
||||
structs::{
|
||||
Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Counter, Date,
|
||||
Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Config, Counter, Date,
|
||||
EmptyAddressData, Height, PartialTxoutData, Price, SentData, Timestamp, TxData, TxoutIndex,
|
||||
},
|
||||
};
|
||||
@@ -28,9 +30,10 @@ pub struct ParseData<'a> {
|
||||
// pub bitcoin_cli: &'a BitcoinCli,
|
||||
pub block: Block,
|
||||
pub block_index: usize,
|
||||
pub config: &'a Config,
|
||||
pub compute_addresses: bool,
|
||||
pub databases: &'a mut Databases,
|
||||
pub datasets: &'a mut AllDatasets,
|
||||
pub datasets: &'a mut Datasets,
|
||||
pub date: Date,
|
||||
pub first_date_height: Height,
|
||||
pub height: Height,
|
||||
@@ -43,6 +46,7 @@ pub fn parse(
|
||||
ParseData {
|
||||
block,
|
||||
block_index,
|
||||
config,
|
||||
compute_addresses,
|
||||
databases,
|
||||
datasets,
|
||||
@@ -56,7 +60,7 @@ pub fn parse(
|
||||
) {
|
||||
// log(&format!("{height}"));
|
||||
|
||||
let timestamp = Timestamp::wrap(block.header.time);
|
||||
let timestamp = Timestamp::from(block.header.time);
|
||||
|
||||
// If false, expect that the code is flawless
|
||||
// or create a 0 value txid database
|
||||
@@ -72,7 +76,7 @@ pub fn parse(
|
||||
let block_price = Price::from_dollar(
|
||||
datasets
|
||||
.price
|
||||
.get_height_ohlc(height, timestamp, previous_timestamp)
|
||||
.get_height_ohlc(height, timestamp, previous_timestamp, config)
|
||||
.unwrap_or_else(|_| panic!("Expect {height} to have a price"))
|
||||
.close as f64,
|
||||
);
|
||||
@@ -106,8 +110,7 @@ pub fn parse(
|
||||
|
||||
let mut block_path_to_sent_data: BTreeMap<BlockPath, SentData> = BTreeMap::default();
|
||||
// let mut received_data: ReceivedData = ReceivedData::default();
|
||||
let mut address_index_to_address_realized_data: BTreeMap<u32, AddressRealizedData> =
|
||||
BTreeMap::default();
|
||||
let mut address_index_to_address_realized_data: BTreeMap<u32, AddressRealizedData> = BTreeMap::default();
|
||||
|
||||
let mut coinbase = Amount::ZERO;
|
||||
let mut satblocks_destroyed = Amount::ZERO;
|
||||
@@ -166,263 +169,228 @@ pub fn parse(
|
||||
)
|
||||
});
|
||||
|
||||
block
|
||||
.txdata
|
||||
.iter()
|
||||
.enumerate()
|
||||
.try_for_each(|(block_tx_index, tx)| {
|
||||
let txid = tx.compute_txid();
|
||||
let tx_index = databases.txid_to_tx_data.metadata.serial as u32;
|
||||
block.txdata.iter().enumerate().try_for_each(|(block_tx_index, tx)| {
|
||||
let txid = tx.compute_txid();
|
||||
let tx_index = databases.txid_to_tx_data.metadata.serial as u32;
|
||||
|
||||
transaction_count += 1;
|
||||
transaction_count += 1;
|
||||
|
||||
// --
|
||||
// outputs
|
||||
// ---
|
||||
// --
|
||||
// outputs
|
||||
// ---
|
||||
|
||||
let mut utxos = BTreeMap::new();
|
||||
let mut spendable_amount = Amount::ZERO;
|
||||
let mut utxos = BTreeMap::new();
|
||||
let mut spendable_amount = Amount::ZERO;
|
||||
|
||||
let is_coinbase = tx.is_coinbase();
|
||||
let is_coinbase = tx.is_coinbase();
|
||||
|
||||
if is_coinbase != (block_tx_index == 0) {
|
||||
unreachable!();
|
||||
}
|
||||
if is_coinbase != (block_tx_index == 0) {
|
||||
unreachable!();
|
||||
}
|
||||
|
||||
let mut inputs_sum = Amount::ZERO;
|
||||
let mut outputs_sum = Amount::ZERO;
|
||||
let mut inputs_sum = Amount::ZERO;
|
||||
let mut outputs_sum = Amount::ZERO;
|
||||
|
||||
let last_block = states.date_data_vec.last_mut_block().unwrap();
|
||||
let last_block = states.date_data_vec.last_mut_block().unwrap();
|
||||
|
||||
// Before `input` to cover outputs being used in the same block as inputs
|
||||
tx.output
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(vout, tx_out)| {
|
||||
if vout > (u16::MAX as usize) {
|
||||
panic!("vout can indeed be bigger than u16::MAX !");
|
||||
}
|
||||
// Before `input` to cover outputs being used in the same block as inputs
|
||||
tx.output
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(vout, tx_out)| {
|
||||
if vout > (u16::MAX as usize) {
|
||||
panic!("vout can indeed be bigger than u16::MAX !");
|
||||
}
|
||||
|
||||
let amount = Amount::wrap(tx_out.value);
|
||||
let amount = Amount::wrap(tx_out.value);
|
||||
|
||||
if is_coinbase {
|
||||
coinbase += amount;
|
||||
} else {
|
||||
outputs_sum += amount;
|
||||
}
|
||||
if is_coinbase {
|
||||
coinbase += amount;
|
||||
} else {
|
||||
outputs_sum += amount;
|
||||
}
|
||||
|
||||
partial_txout_data_vec
|
||||
.pop()
|
||||
.unwrap()
|
||||
// None if not worth parsing (empty/op_return/...)
|
||||
.map(|partial_txout_data| (vout, partial_txout_data))
|
||||
})
|
||||
.for_each(|(vout, partial_txout_data)| {
|
||||
let vout = vout as u16;
|
||||
partial_txout_data_vec
|
||||
.pop()
|
||||
.unwrap()
|
||||
// None if not worth parsing (empty/op_return/...)
|
||||
.map(|partial_txout_data| (vout, partial_txout_data))
|
||||
})
|
||||
.for_each(|(vout, partial_txout_data)| {
|
||||
let vout = vout as u16;
|
||||
|
||||
let txout_index = TxoutIndex::new(tx_index, vout);
|
||||
let txout_index = TxoutIndex::new(tx_index, vout);
|
||||
|
||||
let PartialTxoutData {
|
||||
address,
|
||||
address_index_opt,
|
||||
amount,
|
||||
} = partial_txout_data;
|
||||
let PartialTxoutData {
|
||||
address,
|
||||
address_index_opt,
|
||||
amount,
|
||||
} = partial_txout_data;
|
||||
|
||||
spendable_amount += amount;
|
||||
spendable_amount += amount;
|
||||
|
||||
last_block.receive(amount);
|
||||
last_block.receive(amount);
|
||||
|
||||
utxos.insert(vout, amount);
|
||||
utxos.insert(vout, amount);
|
||||
|
||||
databases.txout_index_to_amount.insert_to_ram(txout_index, amount);
|
||||
|
||||
if compute_addresses {
|
||||
let address = address.unwrap();
|
||||
|
||||
let address_index_to_address_data = address_index_to_address_data.as_mut().unwrap();
|
||||
|
||||
let (address_data, address_index) = {
|
||||
if let Some(address_index) = address_index_opt
|
||||
.or_else(|| databases.address_to_address_index.get_from_ram(&address).cloned())
|
||||
{
|
||||
let address_data = address_index_to_address_data.get_mut(&address_index).unwrap();
|
||||
|
||||
(address_data, address_index)
|
||||
} else {
|
||||
let address_index = databases.address_to_address_index.metadata.serial as u32;
|
||||
|
||||
let address_type = address.to_type();
|
||||
|
||||
if let Some(previous) = databases.address_to_address_index.insert(address, address_index) {
|
||||
dbg!(previous);
|
||||
panic!("address #{address_index} shouldn't be present during put");
|
||||
}
|
||||
|
||||
// Checked new
|
||||
let address_data = address_index_to_address_data
|
||||
.entry(address_index)
|
||||
.and_modify(|_| {
|
||||
panic!("Shouldn't exist");
|
||||
})
|
||||
// Will always insert, it's to avoid insert + get
|
||||
.or_insert(AddressData::new(address_type));
|
||||
|
||||
(address_data, address_index)
|
||||
}
|
||||
};
|
||||
|
||||
// MUST be before received !
|
||||
let address_realized_data = address_index_to_address_realized_data
|
||||
.entry(address_index)
|
||||
.or_insert_with(|| AddressRealizedData::default(address_data));
|
||||
|
||||
address_data.receive(amount, block_price);
|
||||
|
||||
address_realized_data.receive(amount);
|
||||
|
||||
databases
|
||||
.txout_index_to_amount
|
||||
.unsafe_insert(txout_index, amount);
|
||||
.txout_index_to_address_index
|
||||
.insert_to_ram(txout_index, address_index);
|
||||
}
|
||||
});
|
||||
|
||||
if compute_addresses {
|
||||
let address = address.unwrap();
|
||||
if !utxos.is_empty() {
|
||||
databases.txid_to_tx_data.insert(
|
||||
&txid,
|
||||
TxData::new(
|
||||
tx_index,
|
||||
BlockPath::new(date_index as u16, block_index as u16),
|
||||
utxos.len() as u16,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
let address_index_to_address_data =
|
||||
address_index_to_address_data.as_mut().unwrap();
|
||||
// ---
|
||||
// inputs
|
||||
// ---
|
||||
|
||||
let (address_data, address_index) = {
|
||||
if let Some(address_index) = address_index_opt.or_else(|| {
|
||||
databases
|
||||
.address_to_address_index
|
||||
.unsafe_get_from_puts(&address)
|
||||
.cloned()
|
||||
}) {
|
||||
let address_data = address_index_to_address_data
|
||||
.get_mut(&address_index)
|
||||
.unwrap();
|
||||
if !is_coinbase {
|
||||
tx.input.iter().try_for_each(|txin| {
|
||||
let outpoint = txin.previous_output;
|
||||
let input_txid = outpoint.txid;
|
||||
let input_vout = outpoint.vout;
|
||||
|
||||
(address_data, address_index)
|
||||
} else {
|
||||
let address_index =
|
||||
databases.address_to_address_index.metadata.serial as u32;
|
||||
let remove_tx_data_from_cached_puts = {
|
||||
let mut is_tx_data_from_cached_puts = false;
|
||||
|
||||
let address_type = address.to_type();
|
||||
let input_tx_data = txid_to_tx_data.get_mut(&input_txid).unwrap().as_mut().or_else(|| {
|
||||
is_tx_data_from_cached_puts = true;
|
||||
|
||||
if let Some(previous) = databases
|
||||
.address_to_address_index
|
||||
.insert(address, address_index)
|
||||
{
|
||||
dbg!(previous);
|
||||
panic!(
|
||||
"address #{address_index} shouldn't be present during put"
|
||||
);
|
||||
}
|
||||
databases.txid_to_tx_data.get_mut_from_ram(&input_txid)
|
||||
});
|
||||
|
||||
// Checked new
|
||||
let address_data = address_index_to_address_data
|
||||
.entry(address_index)
|
||||
.and_modify(|_| {
|
||||
panic!("Shouldn't exist");
|
||||
})
|
||||
// Will always insert, it's to avoid insert + get
|
||||
.or_insert(AddressData::new(address_type));
|
||||
// Can be none because 0 sats inputs happen
|
||||
// https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
|
||||
|
||||
(address_data, address_index)
|
||||
}
|
||||
};
|
||||
if input_tx_data.is_none() {
|
||||
if !enable_check_if_txout_value_is_zero_in_db
|
||||
|| rpc
|
||||
.get_raw_transaction(&input_txid, None)
|
||||
.unwrap()
|
||||
.output
|
||||
.get(input_vout as usize)
|
||||
.unwrap()
|
||||
.value
|
||||
.to_sat()
|
||||
== 0
|
||||
{
|
||||
return ControlFlow::Continue::<()>(());
|
||||
}
|
||||
|
||||
// MUST be before received !
|
||||
let address_realized_data = address_index_to_address_realized_data
|
||||
.entry(address_index)
|
||||
.or_insert_with(|| AddressRealizedData::default(address_data));
|
||||
|
||||
address_data.receive(amount, block_price);
|
||||
|
||||
address_realized_data.receive(amount);
|
||||
|
||||
databases
|
||||
.txout_index_to_address_index
|
||||
.unsafe_insert(txout_index, address_index);
|
||||
dbg!((input_txid, txid, tx_index, input_vout));
|
||||
panic!("Txid to be in txid_to_tx_data");
|
||||
}
|
||||
});
|
||||
|
||||
if !utxos.is_empty() {
|
||||
databases.txid_to_tx_data.insert(
|
||||
&txid,
|
||||
TxData::new(
|
||||
tx_index,
|
||||
BlockPath::new(date_index as u16, block_index as u16),
|
||||
utxos.len() as u16,
|
||||
),
|
||||
);
|
||||
}
|
||||
let input_tx_data = input_tx_data.unwrap();
|
||||
let input_tx_index = input_tx_data.index;
|
||||
let input_vout = input_vout as u16;
|
||||
let input_txout_index = TxoutIndex::new(input_tx_index, input_vout);
|
||||
|
||||
// ---
|
||||
// inputs
|
||||
// ---
|
||||
// if input_tx_index == 2516 || input_tx_index == 2490 {
|
||||
// dbg!(input_tx_index, &input_tx_data.utxos);
|
||||
// }
|
||||
|
||||
if !is_coinbase {
|
||||
tx.input.iter().try_for_each(|txin| {
|
||||
let outpoint = txin.previous_output;
|
||||
let input_txid = outpoint.txid;
|
||||
let input_vout = outpoint.vout;
|
||||
// let input_amount = input_tx_data.utxos.remove(&input_vout);
|
||||
|
||||
let remove_tx_data_from_cached_puts = {
|
||||
let mut is_tx_data_from_cached_puts = false;
|
||||
let input_amount_and_address_index = databases
|
||||
.txout_index_to_amount
|
||||
.remove(&input_txout_index)
|
||||
.map(|amount| {
|
||||
(
|
||||
amount,
|
||||
databases.txout_index_to_address_index.remove(&input_txout_index),
|
||||
)
|
||||
}) // Remove from cached puts
|
||||
.or_else(|| txout_index_to_amount_and_address_index.remove(&input_txout_index));
|
||||
|
||||
let input_tx_data = txid_to_tx_data
|
||||
.get_mut(&input_txid)
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.or_else(|| {
|
||||
is_tx_data_from_cached_puts = true;
|
||||
|
||||
databases
|
||||
.txid_to_tx_data
|
||||
.unsafe_get_mut_from_puts(&input_txid)
|
||||
});
|
||||
|
||||
// Can be none because 0 sats inputs happen
|
||||
// https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
|
||||
|
||||
if input_tx_data.is_none() {
|
||||
if !enable_check_if_txout_value_is_zero_in_db
|
||||
|| rpc
|
||||
.get_raw_transaction(&input_txid, None)
|
||||
.unwrap()
|
||||
.output
|
||||
.get(input_vout as usize)
|
||||
.unwrap()
|
||||
.value
|
||||
.to_sat()
|
||||
== 0
|
||||
{
|
||||
return ControlFlow::Continue::<()>(());
|
||||
}
|
||||
|
||||
dbg!((input_txid, txid, tx_index, input_vout));
|
||||
panic!("Txid to be in txid_to_tx_data");
|
||||
if input_amount_and_address_index.is_none() {
|
||||
if !enable_check_if_txout_value_is_zero_in_db
|
||||
|| rpc
|
||||
.get_raw_transaction(&input_txid, None)
|
||||
.unwrap()
|
||||
.output
|
||||
.get(input_vout as usize)
|
||||
.unwrap()
|
||||
.value
|
||||
.to_sat()
|
||||
== 0
|
||||
{
|
||||
return ControlFlow::Continue::<()>(());
|
||||
}
|
||||
|
||||
let input_tx_data = input_tx_data.unwrap();
|
||||
let input_tx_index = input_tx_data.index;
|
||||
let input_vout = input_vout as u16;
|
||||
let input_txout_index = TxoutIndex::new(input_tx_index, input_vout);
|
||||
dbg!((input_txid, tx_index, input_tx_index, input_vout, input_tx_data, txid,));
|
||||
panic!("Txout index to be in txout_index_to_txout_value");
|
||||
}
|
||||
|
||||
// if input_tx_index == 2516 || input_tx_index == 2490 {
|
||||
// dbg!(input_tx_index, &input_tx_data.utxos);
|
||||
// }
|
||||
input_tx_data.utxos -= 1;
|
||||
|
||||
// let input_amount = input_tx_data.utxos.remove(&input_vout);
|
||||
let (input_amount, input_address_index) = input_amount_and_address_index.unwrap();
|
||||
|
||||
let input_amount_and_address_index = databases
|
||||
.txout_index_to_amount
|
||||
.remove(&input_txout_index)
|
||||
.map(|amount| {
|
||||
(
|
||||
amount,
|
||||
databases
|
||||
.txout_index_to_address_index
|
||||
.remove(&input_txout_index),
|
||||
)
|
||||
}) // Remove from cached puts
|
||||
.or_else(|| {
|
||||
txout_index_to_amount_and_address_index.remove(&input_txout_index)
|
||||
});
|
||||
let input_block_path = input_tx_data.block_path;
|
||||
|
||||
if input_amount_and_address_index.is_none() {
|
||||
if !enable_check_if_txout_value_is_zero_in_db
|
||||
|| rpc
|
||||
.get_raw_transaction(&input_txid, None)
|
||||
.unwrap()
|
||||
.output
|
||||
.get(input_vout as usize)
|
||||
.unwrap()
|
||||
.value
|
||||
.to_sat()
|
||||
== 0
|
||||
{
|
||||
return ControlFlow::Continue::<()>(());
|
||||
}
|
||||
let BlockPath {
|
||||
date_index: input_date_index,
|
||||
block_index: input_block_index,
|
||||
} = input_block_path;
|
||||
|
||||
dbg!((
|
||||
input_txid,
|
||||
tx_index,
|
||||
input_tx_index,
|
||||
input_vout,
|
||||
input_tx_data,
|
||||
txid,
|
||||
));
|
||||
panic!("Txout index to be in txout_index_to_txout_value");
|
||||
}
|
||||
|
||||
input_tx_data.utxos -= 1;
|
||||
|
||||
let (input_amount, input_address_index) =
|
||||
input_amount_and_address_index.unwrap();
|
||||
|
||||
let input_block_path = input_tx_data.block_path;
|
||||
|
||||
let BlockPath {
|
||||
date_index: input_date_index,
|
||||
block_index: input_block_index,
|
||||
} = input_block_path;
|
||||
|
||||
let input_date_data = states
|
||||
let input_date_data =
|
||||
states
|
||||
.date_data_vec
|
||||
.get_mut(input_date_index as usize)
|
||||
.unwrap_or_else(|| {
|
||||
@@ -430,121 +398,112 @@ pub fn parse(
|
||||
panic!()
|
||||
});
|
||||
|
||||
let input_block_data = input_date_data
|
||||
.blocks
|
||||
.get_mut(input_block_index as usize)
|
||||
let input_block_data = input_date_data
|
||||
.blocks
|
||||
.get_mut(input_block_index as usize)
|
||||
.unwrap_or_else(|| {
|
||||
dbg!(
|
||||
height,
|
||||
&input_txid,
|
||||
input_block_path,
|
||||
input_date_index,
|
||||
input_block_index,
|
||||
);
|
||||
panic!()
|
||||
});
|
||||
|
||||
input_block_data.send(input_amount);
|
||||
|
||||
inputs_sum += input_amount;
|
||||
|
||||
block_path_to_sent_data
|
||||
.entry(input_block_path)
|
||||
.or_default()
|
||||
.send(input_amount);
|
||||
|
||||
satblocks_destroyed += input_amount * (height - input_block_data.height);
|
||||
|
||||
satdays_destroyed +=
|
||||
input_amount * date.signed_duration_since(*input_date_data.date).num_days() as u64;
|
||||
|
||||
if compute_addresses {
|
||||
let input_address_index = input_address_index.unwrap_or_else(|| {
|
||||
dbg!(
|
||||
height,
|
||||
input_amount,
|
||||
&input_tx_data,
|
||||
input_address_index,
|
||||
input_txout_index,
|
||||
txid,
|
||||
input_txid,
|
||||
input_vout
|
||||
);
|
||||
panic!()
|
||||
});
|
||||
|
||||
let address_index_to_address_data = address_index_to_address_data.as_mut().unwrap();
|
||||
|
||||
let input_address_data = address_index_to_address_data
|
||||
.get_mut(&input_address_index)
|
||||
.unwrap_or_else(|| {
|
||||
dbg!(
|
||||
height,
|
||||
&input_txid,
|
||||
input_block_path,
|
||||
input_date_index,
|
||||
input_block_index,
|
||||
);
|
||||
panic!()
|
||||
dbg!(input_address_index, input_txout_index, input_txid, input_vout);
|
||||
panic!();
|
||||
});
|
||||
|
||||
input_block_data.send(input_amount);
|
||||
let input_address_realized_data = address_index_to_address_realized_data
|
||||
.entry(input_address_index)
|
||||
.or_insert_with(|| AddressRealizedData::default(input_address_data));
|
||||
|
||||
inputs_sum += input_amount;
|
||||
let previous_price = input_block_data.price;
|
||||
|
||||
block_path_to_sent_data
|
||||
.entry(input_block_path)
|
||||
.or_default()
|
||||
.send(input_amount);
|
||||
|
||||
satblocks_destroyed += input_amount * (height - input_block_data.height);
|
||||
|
||||
satdays_destroyed += input_amount
|
||||
* date.signed_duration_since(*input_date_data.date).num_days() as u64;
|
||||
|
||||
if compute_addresses {
|
||||
let input_address_index = input_address_index.unwrap_or_else(|| {
|
||||
// MUST be after `or_insert_with`
|
||||
input_address_data
|
||||
.send(input_amount, previous_price)
|
||||
.unwrap_or_else(|_| {
|
||||
dbg!(
|
||||
height,
|
||||
input_amount,
|
||||
&input_tx_data,
|
||||
input_address_index,
|
||||
input_txout_index,
|
||||
txid,
|
||||
input_txid,
|
||||
input_vout
|
||||
input_amount,
|
||||
tx_index,
|
||||
input_tx_index,
|
||||
input_vout,
|
||||
&input_address_data
|
||||
);
|
||||
|
||||
panic!()
|
||||
});
|
||||
|
||||
let address_index_to_address_data =
|
||||
address_index_to_address_data.as_mut().unwrap();
|
||||
|
||||
let input_address_data = address_index_to_address_data
|
||||
.get_mut(&input_address_index)
|
||||
.unwrap_or_else(|| {
|
||||
dbg!(
|
||||
input_address_index,
|
||||
input_txout_index,
|
||||
input_txid,
|
||||
input_vout
|
||||
);
|
||||
panic!();
|
||||
});
|
||||
|
||||
let input_address_realized_data =
|
||||
address_index_to_address_realized_data
|
||||
.entry(input_address_index)
|
||||
.or_insert_with(|| {
|
||||
AddressRealizedData::default(input_address_data)
|
||||
});
|
||||
|
||||
let previous_price = input_block_data.price;
|
||||
|
||||
// MUST be after `or_insert_with`
|
||||
input_address_data
|
||||
.send(input_amount, previous_price)
|
||||
.unwrap_or_else(|_| {
|
||||
dbg!(
|
||||
input_address_index,
|
||||
txid,
|
||||
input_txid,
|
||||
input_amount,
|
||||
tx_index,
|
||||
input_tx_index,
|
||||
input_vout,
|
||||
&input_address_data
|
||||
);
|
||||
|
||||
panic!()
|
||||
});
|
||||
|
||||
input_address_realized_data.send(
|
||||
input_amount,
|
||||
block_price,
|
||||
previous_price,
|
||||
timestamp,
|
||||
input_block_data.timestamp,
|
||||
);
|
||||
};
|
||||
|
||||
is_tx_data_from_cached_puts && input_tx_data.is_empty()
|
||||
input_address_realized_data.send(
|
||||
input_amount,
|
||||
block_price,
|
||||
previous_price,
|
||||
timestamp,
|
||||
input_block_data.timestamp,
|
||||
);
|
||||
};
|
||||
|
||||
if remove_tx_data_from_cached_puts {
|
||||
// Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
|
||||
databases.txid_to_tx_data.remove_from_puts(&input_txid)
|
||||
}
|
||||
is_tx_data_from_cached_puts && input_tx_data.is_empty()
|
||||
};
|
||||
|
||||
ControlFlow::Continue(())
|
||||
})?;
|
||||
}
|
||||
if remove_tx_data_from_cached_puts {
|
||||
// Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
|
||||
databases.txid_to_tx_data.remove_from_ram(&input_txid)
|
||||
}
|
||||
|
||||
amount_sent += inputs_sum;
|
||||
ControlFlow::Continue(())
|
||||
})?;
|
||||
}
|
||||
|
||||
let fee = inputs_sum - outputs_sum;
|
||||
amount_sent += inputs_sum;
|
||||
|
||||
fees_total += fee;
|
||||
fees.push(fee);
|
||||
let fee = inputs_sum - outputs_sum;
|
||||
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
fees_total += fee;
|
||||
fees.push(fee);
|
||||
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
|
||||
if !partial_txout_data_vec.is_empty() {
|
||||
panic!("partial_txout_data_vec should've been fully consumed");
|
||||
@@ -553,7 +512,7 @@ pub fn parse(
|
||||
txid_to_tx_data.into_iter().for_each(|(txid, tx_data)| {
|
||||
if let Some(tx_data) = tx_data {
|
||||
if tx_data.is_empty() {
|
||||
databases.txid_to_tx_data.remove_from_db(txid);
|
||||
databases.txid_to_tx_data.remove_later_from_disk(txid);
|
||||
} else {
|
||||
databases.txid_to_tx_data.update(txid, tx_data);
|
||||
}
|
||||
@@ -577,24 +536,17 @@ pub fn parse(
|
||||
|
||||
if datasets.utxo.needs_durable_states(height, date) {
|
||||
if let Some(previous_last_block_data) = previous_last_block_data {
|
||||
block_path_to_sent_data
|
||||
.iter()
|
||||
.for_each(|(block_path, sent_data)| {
|
||||
let block_data =
|
||||
states.date_data_vec.get_block_data(block_path).unwrap();
|
||||
block_path_to_sent_data.iter().for_each(|(block_path, sent_data)| {
|
||||
let block_data = states.date_data_vec.get_block_data(block_path).unwrap();
|
||||
|
||||
if block_data.height != height {
|
||||
states
|
||||
.utxo_cohorts_durable_states
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.subtract_moved(
|
||||
block_data,
|
||||
sent_data,
|
||||
previous_last_block_data,
|
||||
);
|
||||
}
|
||||
});
|
||||
if block_data.height != height {
|
||||
states.utxo_cohorts_durable_states.as_mut().unwrap().subtract_moved(
|
||||
block_data,
|
||||
sent_data,
|
||||
previous_last_block_data,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let last_block_data = states.date_data_vec.last_block().unwrap();
|
||||
@@ -612,11 +564,7 @@ pub fn parse(
|
||||
.utxo_cohorts_durable_states
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.udpate_age_if_needed(
|
||||
block_data,
|
||||
last_block_data,
|
||||
previous_last_block_data,
|
||||
);
|
||||
.udpate_age_if_needed(block_data, last_block_data, previous_last_block_data);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -625,14 +573,7 @@ pub fn parse(
|
||||
.utxo_cohorts_durable_states
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.compute_one_shot_states(
|
||||
block_price,
|
||||
if is_date_last_block {
|
||||
Some(date_price)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
);
|
||||
.compute_one_shot_states(block_price, if is_date_last_block { Some(date_price) } else { None });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -665,10 +606,10 @@ pub fn parse(
|
||||
// TODO: Only compute if needed
|
||||
address_cohorts_output_states.replace(AddressCohortsOutputStates::default());
|
||||
|
||||
address_index_to_address_realized_data.iter().for_each(
|
||||
|(address_index, address_realized_data)| {
|
||||
let current_address_data =
|
||||
address_index_to_address_data.get(address_index).unwrap();
|
||||
address_index_to_address_realized_data
|
||||
.iter()
|
||||
.for_each(|(address_index, address_realized_data)| {
|
||||
let current_address_data = address_index_to_address_data.get(address_index).unwrap();
|
||||
|
||||
states
|
||||
.address_cohorts_durable_states
|
||||
@@ -708,42 +649,34 @@ pub fn parse(
|
||||
¤t_address_data.compute_liquidity_classification(),
|
||||
)
|
||||
.unwrap();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
address_cohorts_one_shot_states.replace(
|
||||
states
|
||||
.address_cohorts_durable_states
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.compute_one_shot_states(
|
||||
block_price,
|
||||
if is_date_last_block {
|
||||
Some(date_price)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
),
|
||||
.compute_one_shot_states(block_price, if is_date_last_block { Some(date_price) } else { None }),
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if compute_addresses {
|
||||
address_index_to_address_data.unwrap().into_iter().for_each(
|
||||
|(address_index, address_data)| {
|
||||
address_index_to_address_data
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.for_each(|(address_index, address_data)| {
|
||||
if address_data.is_empty() {
|
||||
databases.address_index_to_empty_address_data.unsafe_insert(
|
||||
address_index,
|
||||
EmptyAddressData::from_non_empty(&address_data),
|
||||
);
|
||||
databases
|
||||
.address_index_to_empty_address_data
|
||||
.insert_to_ram(address_index, EmptyAddressData::from_non_empty(&address_data));
|
||||
} else {
|
||||
databases
|
||||
.address_index_to_address_data
|
||||
.unsafe_insert(address_index, address_data);
|
||||
.insert_to_ram(address_index, address_data);
|
||||
}
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
datasets.insert(InsertData {
|
||||
@@ -848,17 +781,15 @@ fn prepare_outputs(
|
||||
.collect_vec();
|
||||
|
||||
if compute_addresses {
|
||||
partial_txout_data_vec
|
||||
.par_iter_mut()
|
||||
.for_each(|partial_tx_out_data| {
|
||||
if let Some(partial_tx_out_data) = partial_tx_out_data {
|
||||
let address_index_opt = address_to_address_index
|
||||
.unsafe_get(partial_tx_out_data.address.as_ref().unwrap())
|
||||
.cloned();
|
||||
partial_txout_data_vec.par_iter_mut().for_each(|partial_tx_out_data| {
|
||||
if let Some(partial_tx_out_data) = partial_tx_out_data {
|
||||
let address_index_opt = address_to_address_index
|
||||
.unsafe_get(partial_tx_out_data.address.as_ref().unwrap())
|
||||
.cloned();
|
||||
|
||||
partial_tx_out_data.address_index_opt = address_index_opt;
|
||||
}
|
||||
});
|
||||
partial_tx_out_data.address_index_opt = address_index_opt;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
TxoutsParsingResults {
|
||||
@@ -896,7 +827,7 @@ fn prepare_inputs<'a>(
|
||||
|
||||
let mut tx_datas = txid_to_tx_data
|
||||
.par_iter()
|
||||
.map(|(txid, _)| txid_to_tx_data_db.unsafe_get(txid))
|
||||
.map(|(txid, _)| txid_to_tx_data_db.get(txid))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
txid_to_tx_data.values_mut().rev().for_each(|tx_data_opt| {
|
||||
@@ -934,11 +865,8 @@ fn prepare_inputs<'a>(
|
||||
// https://mempool.space/tx/9d8a0d851c9fb2cdf1c6d9406ce97e19e6911ae3503ab2dd5f38640bacdac996
|
||||
// which is used later as input
|
||||
.map(|amount| {
|
||||
let address_index = compute_addresses.then(|| {
|
||||
*txout_index_to_address_index_db
|
||||
.unsafe_get(&txout_index)
|
||||
.unwrap()
|
||||
});
|
||||
let address_index =
|
||||
compute_addresses.then(|| *txout_index_to_address_index_db.unsafe_get(&txout_index).unwrap());
|
||||
|
||||
(txout_index, (*amount, address_index))
|
||||
})
|
||||
@@ -986,21 +914,16 @@ fn compute_address_index_to_address_data(
|
||||
address_index_to_address_data
|
||||
.par_iter_mut()
|
||||
.for_each(|(address_index, address_data)| {
|
||||
if let Some(_address_data) =
|
||||
address_index_to_address_data_db.unsafe_get_from_cache(address_index)
|
||||
{
|
||||
if let Some(_address_data) = address_index_to_address_data_db.get_from_ram(address_index) {
|
||||
_address_data.clone_into(address_data);
|
||||
} else if let Some(empty_address_data) =
|
||||
address_index_to_empty_address_data_db.unsafe_get_from_cache(address_index)
|
||||
} else if let Some(empty_address_data) = address_index_to_empty_address_data_db.get_from_ram(address_index)
|
||||
{
|
||||
*address_data = AddressData::from_empty(empty_address_data);
|
||||
} else if let Some(_address_data) =
|
||||
address_index_to_address_data_db.unsafe_get_from_db(address_index)
|
||||
{
|
||||
} else if let Some(_address_data) = address_index_to_address_data_db.get_from_disk(address_index) {
|
||||
_address_data.clone_into(address_data);
|
||||
} else {
|
||||
let empty_address_data = address_index_to_empty_address_data_db
|
||||
.unsafe_get_from_db(address_index)
|
||||
.get_from_disk(address_index)
|
||||
.unwrap();
|
||||
|
||||
*address_data = AddressData::from_empty(empty_address_data);
|
||||
@@ -0,0 +1,60 @@
|
||||
use std::{fs, io, path::Path};
|
||||
|
||||
use log::info;
|
||||
use snkrj::AnyDatabase;
|
||||
|
||||
use crate::structs::{Config, Date, Height};
|
||||
|
||||
use super::Metadata;
|
||||
|
||||
pub trait AnyDatabaseGroup
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
fn init(config: &Config) -> Self {
|
||||
let s = Self::import(config);
|
||||
s.create_dir_all().unwrap();
|
||||
s
|
||||
}
|
||||
|
||||
fn import(config: &Config) -> Self;
|
||||
|
||||
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>>;
|
||||
fn open_all(&mut self);
|
||||
|
||||
fn metadata(&mut self) -> &mut Metadata;
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata().export(height, date)
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(self.path())
|
||||
}
|
||||
|
||||
fn remove_dir_all(&self) -> color_eyre::Result<(), io::Error> {
|
||||
fs::remove_dir_all(self.path())
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
|
||||
info!(
|
||||
"Reset {}",
|
||||
self.path()
|
||||
.components()
|
||||
.last()
|
||||
.unwrap()
|
||||
.as_os_str()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
);
|
||||
|
||||
self.reset_metadata();
|
||||
self.remove_dir_all()?;
|
||||
self.create_dir_all()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self);
|
||||
|
||||
fn path(&self) -> &Path;
|
||||
}
|
||||
+43
-43
@@ -2,19 +2,20 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::{
|
||||
states::AddressCohortsDurableStates,
|
||||
structs::{AddressData, Date, Height},
|
||||
utils::time,
|
||||
parser::states::AddressCohortsDurableStates,
|
||||
structs::{AddressData, Config},
|
||||
};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = u32;
|
||||
type Value = AddressData;
|
||||
@@ -22,8 +23,9 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AddressIndexToAddressData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
#[allocative(skip)]
|
||||
pub map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -44,10 +46,10 @@ impl DerefMut for AddressIndexToAddressData {
|
||||
pub const ADDRESS_INDEX_DB_MAX_SIZE: usize = 250_000;
|
||||
|
||||
impl AddressIndexToAddressData {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
@@ -58,20 +60,21 @@ impl AddressIndexToAddressData {
|
||||
|
||||
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
|
||||
/// Though it makes it easy to use with rayon.
|
||||
pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index).unwrap().get_from_puts(key)
|
||||
self.get(&db_index).unwrap().get_from_ram(key)
|
||||
}
|
||||
|
||||
pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index).unwrap().db_get(key)
|
||||
self.get(&db_index).unwrap().get_from_disk(key)
|
||||
}
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path().to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -80,31 +83,31 @@ impl AddressIndexToAddressData {
|
||||
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_addres_cohorts_durable_states(&mut self) -> AddressCohortsDurableStates {
|
||||
time("Iter through address_index_to_address_data", || {
|
||||
self.open_all();
|
||||
// time("Iter through address_index_to_address_data", || {
|
||||
self.open_all();
|
||||
|
||||
// MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
|
||||
mem::take(&mut self.map)
|
||||
.par_iter()
|
||||
.map(|(_, database)| {
|
||||
let mut s = AddressCohortsDurableStates::default();
|
||||
// MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
|
||||
mem::take(&mut self.map)
|
||||
.par_iter()
|
||||
.map(|(_, database)| {
|
||||
let mut s = AddressCohortsDurableStates::default();
|
||||
|
||||
database
|
||||
.iter()
|
||||
.map(|r| r.unwrap().1)
|
||||
.for_each(|address_data| s.increment(address_data).unwrap());
|
||||
database
|
||||
.iter_disk()
|
||||
.map(|r| r.unwrap().1)
|
||||
.for_each(|address_data| s.increment(address_data).unwrap());
|
||||
|
||||
s
|
||||
})
|
||||
.sum()
|
||||
})
|
||||
s
|
||||
})
|
||||
.sum()
|
||||
// })
|
||||
}
|
||||
|
||||
fn db_index(key: &Key) -> usize {
|
||||
@@ -113,30 +116,23 @@ impl AddressIndexToAddressData {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for AddressIndexToAddressData {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config
|
||||
.path_databases()
|
||||
.join("address_index_to_address_data");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"address_index_to_address_data"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -167,7 +163,11 @@ impl AnyDatabaseGroup for AddressIndexToAddressData {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+27
-28
@@ -2,16 +2,16 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Date, EmptyAddressData, Height};
|
||||
use crate::structs::{Config, EmptyAddressData};
|
||||
|
||||
use super::{
|
||||
AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata, ADDRESS_INDEX_DB_MAX_SIZE,
|
||||
};
|
||||
use super::{AnyDatabaseGroup, Metadata, ADDRESS_INDEX_DB_MAX_SIZE};
|
||||
|
||||
type Key = u32;
|
||||
type Value = EmptyAddressData;
|
||||
@@ -19,8 +19,9 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AddressIndexToEmptyAddressData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -39,10 +40,10 @@ impl DerefMut for AddressIndexToEmptyAddressData {
|
||||
}
|
||||
|
||||
impl AddressIndexToEmptyAddressData {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
@@ -53,13 +54,13 @@ impl AddressIndexToEmptyAddressData {
|
||||
|
||||
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed
|
||||
/// Though it makes it easy to use with rayon.
|
||||
pub fn unsafe_get_from_cache(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index).and_then(|db| db.get_from_puts(key))
|
||||
self.get(&db_index).and_then(|db| db.get_from_ram(key))
|
||||
}
|
||||
|
||||
pub fn unsafe_get_from_db(&self, key: &Key) -> Option<&Value> {
|
||||
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
|
||||
let db_index = Self::db_index(key);
|
||||
|
||||
self.get(&db_index)
|
||||
@@ -67,11 +68,12 @@ impl AddressIndexToEmptyAddressData {
|
||||
dbg!(&self.map.keys(), &key, &db_index);
|
||||
panic!()
|
||||
})
|
||||
.db_get(key)
|
||||
.get_from_disk(key)
|
||||
}
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path.to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -80,7 +82,7 @@ impl AddressIndexToEmptyAddressData {
|
||||
(db_index + 1) * ADDRESS_INDEX_DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
@@ -92,30 +94,23 @@ impl AddressIndexToEmptyAddressData {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config
|
||||
.path_databases()
|
||||
.join("address_index_to_empty_address_data");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"address_index_to_empty_address_data"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -146,7 +141,11 @@ impl AnyDatabaseGroup for AddressIndexToEmptyAddressData {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+72
-57
@@ -6,10 +6,11 @@ use std::{
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database};
|
||||
|
||||
use crate::structs::{Address, Date, Height, U8x19, U8x31};
|
||||
use crate::structs::{Address, Config, U8x19, U8x31};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Value = u32;
|
||||
type U8x19Database = Database<U8x19, Value>;
|
||||
@@ -30,18 +31,30 @@ type MultisigDatabase = U32Database;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AddressToAddressIndex {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
#[allocative(skip)]
|
||||
p2pk: BTreeMap<u16, P2PKDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2pkh: BTreeMap<u16, P2PKHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2sh: BTreeMap<u16, P2SHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2wpkh: BTreeMap<u16, P2WPKHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2wsh: BTreeMap<u16, P2WSHDatabase>,
|
||||
#[allocative(skip)]
|
||||
p2tr: BTreeMap<u16, P2TRDatabase>,
|
||||
#[allocative(skip)]
|
||||
op_return: Option<OpReturnDatabase>,
|
||||
#[allocative(skip)]
|
||||
push_only: Option<PushOnlyDatabase>,
|
||||
#[allocative(skip)]
|
||||
unknown: Option<UnknownDatabase>,
|
||||
#[allocative(skip)]
|
||||
empty: Option<EmptyDatabase>,
|
||||
#[allocative(skip)]
|
||||
multisig: Option<MultisigDatabase>,
|
||||
}
|
||||
|
||||
@@ -102,19 +115,19 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unsafe_get_from_puts(&self, address: &Address) -> Option<&Value> {
|
||||
pub fn get_from_ram(&self, address: &Address) -> Option<&Value> {
|
||||
match address {
|
||||
Address::Empty(key) => self.empty.as_ref().unwrap().get_from_puts(key),
|
||||
Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_puts(key),
|
||||
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_puts(key),
|
||||
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_puts(key),
|
||||
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_puts(key),
|
||||
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_puts(key),
|
||||
Address::Empty(key) => self.empty.as_ref().unwrap().get_from_ram(key),
|
||||
Address::Unknown(key) => self.unknown.as_ref().unwrap().get_from_ram(key),
|
||||
Address::OpReturn(key) => self.op_return.as_ref().unwrap().get_from_ram(key),
|
||||
Address::PushOnly(key) => self.push_only.as_ref().unwrap().get_from_ram(key),
|
||||
Address::MultiSig(key) => self.multisig.as_ref().unwrap().get_from_ram(key),
|
||||
Address::P2PK((prefix, key)) => self.p2pk.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2PKH((prefix, key)) => self.p2pkh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2SH((prefix, key)) => self.p2sh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2WPKH((prefix, key)) => self.p2wpkh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2WSH((prefix, key)) => self.p2wsh.get(prefix).unwrap().get_from_ram(key),
|
||||
Address::P2TR((prefix, key)) => self.p2tr.get(prefix).unwrap().get_from_ram(key),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,12 +173,12 @@ impl AddressToAddressIndex {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn path_p2pk() -> PathBuf {
|
||||
Self::root().join("p2pk")
|
||||
fn path_p2pk(&self) -> PathBuf {
|
||||
self.path().join("p2pk")
|
||||
}
|
||||
|
||||
pub fn open_p2pk(&mut self, prefix: u16) -> &mut P2PKDatabase {
|
||||
let path = Self::path_p2pk();
|
||||
let path = self.path_p2pk();
|
||||
self.p2pk.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
Database::open(path).unwrap()
|
||||
@@ -173,7 +186,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2pk(&mut self) {
|
||||
let path = Self::path_p2pk();
|
||||
let path = self.path_p2pk();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -184,12 +197,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2pkh() -> PathBuf {
|
||||
Self::root().join("p2pkh")
|
||||
fn path_p2pkh(&self) -> PathBuf {
|
||||
self.path().join("p2pkh")
|
||||
}
|
||||
|
||||
pub fn open_p2pkh(&mut self, prefix: u16) -> &mut P2PKHDatabase {
|
||||
let path = Self::path_p2pkh();
|
||||
let path = self.path_p2pkh();
|
||||
|
||||
self.p2pkh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -198,7 +211,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2pkh(&mut self) {
|
||||
let path = Self::path_p2pkh();
|
||||
let path = self.path_p2pkh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -209,12 +222,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2sh() -> PathBuf {
|
||||
Self::root().join("p2sh")
|
||||
fn path_p2sh(&self) -> PathBuf {
|
||||
self.path().join("p2sh")
|
||||
}
|
||||
|
||||
pub fn open_p2sh(&mut self, prefix: u16) -> &mut P2SHDatabase {
|
||||
let path = Self::path_p2sh();
|
||||
let path = self.path_p2sh();
|
||||
|
||||
self.p2sh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -223,7 +236,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2sh(&mut self) {
|
||||
let path = Self::path_p2sh();
|
||||
let path = self.path_p2sh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -234,12 +247,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2wpkh() -> PathBuf {
|
||||
Self::root().join("p2wpkh")
|
||||
fn path_p2wpkh(&self) -> PathBuf {
|
||||
self.path().join("p2wpkh")
|
||||
}
|
||||
|
||||
pub fn open_p2wpkh(&mut self, prefix: u16) -> &mut P2WPKHDatabase {
|
||||
let path = Self::path_p2wpkh();
|
||||
let path = self.path_p2wpkh();
|
||||
|
||||
self.p2wpkh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -248,7 +261,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2wpkh(&mut self) {
|
||||
let path = Self::path_p2wpkh();
|
||||
let path = self.path_p2wpkh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -259,12 +272,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2wsh() -> PathBuf {
|
||||
Self::root().join("p2wsh")
|
||||
fn path_p2wsh(&self) -> PathBuf {
|
||||
self.path().join("p2wsh")
|
||||
}
|
||||
|
||||
pub fn open_p2wsh(&mut self, prefix: u16) -> &mut P2WSHDatabase {
|
||||
let path = Self::path_p2wsh();
|
||||
let path = self.path_p2wsh();
|
||||
|
||||
self.p2wsh.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -273,7 +286,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2wsh(&mut self) {
|
||||
let path = Self::path_p2wsh();
|
||||
let path = self.path_p2wsh();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -284,12 +297,12 @@ impl AddressToAddressIndex {
|
||||
});
|
||||
}
|
||||
|
||||
fn path_p2tr() -> PathBuf {
|
||||
Self::root().join("p2tr")
|
||||
fn path_p2tr(&self) -> PathBuf {
|
||||
self.path().join("p2tr")
|
||||
}
|
||||
|
||||
pub fn open_p2tr(&mut self, prefix: u16) -> &mut P2TRDatabase {
|
||||
let path = Self::path_p2tr();
|
||||
let path = self.path_p2tr();
|
||||
|
||||
self.p2tr.entry(prefix).or_insert_with(|| {
|
||||
let path = path.join(prefix.to_string());
|
||||
@@ -298,7 +311,7 @@ impl AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn open_all_p2tr(&mut self) {
|
||||
let path = Self::path_p2tr();
|
||||
let path = self.path_p2tr();
|
||||
Self::path_to_group_prefixes(&path)
|
||||
.into_iter()
|
||||
.for_each(|prefix| {
|
||||
@@ -311,34 +324,36 @@ impl AddressToAddressIndex {
|
||||
|
||||
pub fn open_unknown(&mut self) -> &mut UnknownDatabase {
|
||||
self.unknown
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("unknown")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("unknown")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_op_return(&mut self) -> &mut UnknownDatabase {
|
||||
self.op_return
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("op_return")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("op_return")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_push_only(&mut self) -> &mut UnknownDatabase {
|
||||
self.push_only
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("push_only")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("push_only")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_empty(&mut self) -> &mut UnknownDatabase {
|
||||
self.empty
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("empty")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("empty")).unwrap())
|
||||
}
|
||||
|
||||
pub fn open_multisig(&mut self) -> &mut MultisigDatabase {
|
||||
self.multisig
|
||||
.get_or_insert_with(|| Database::open(Self::root().join("multisig")).unwrap())
|
||||
.get_or_insert_with(|| Database::open(self.path.join("multisig")).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for AddressToAddressIndex {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("address_to_address_index");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
|
||||
p2pk: BTreeMap::default(),
|
||||
p2pkh: BTreeMap::default(),
|
||||
@@ -355,22 +370,18 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::path_p2pk()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2pkh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2sh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2wpkh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2wsh()).unwrap();
|
||||
fs::create_dir_all(Self::path_p2tr())
|
||||
fs::create_dir_all(self.path_p2pk()).unwrap();
|
||||
fs::create_dir_all(self.path_p2pkh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2sh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2wpkh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2wsh()).unwrap();
|
||||
fs::create_dir_all(self.path_p2tr())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset()
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"address_to_address_index"
|
||||
}
|
||||
|
||||
fn drain_to_vec(&mut self) -> Vec<Box<dyn AnyDatabase + Send>> {
|
||||
mem::take(&mut self.p2pk)
|
||||
.into_values()
|
||||
@@ -424,7 +435,11 @@ impl AnyDatabaseGroup for AddressToAddressIndex {
|
||||
self.open_all_p2tr();
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,8 @@ use std::{
|
||||
};
|
||||
|
||||
use crate::{
|
||||
io::Serialization,
|
||||
structs::{Counter, Date, Height},
|
||||
Serialization,
|
||||
};
|
||||
|
||||
#[derive(Default, Debug, Encode, Decode, Allocative)]
|
||||
@@ -35,10 +35,10 @@ impl DerefMut for Metadata {
|
||||
}
|
||||
|
||||
impl Metadata {
|
||||
pub fn import(path: PathBuf, version: u16) -> Self {
|
||||
pub fn import(path: &Path, version: u16) -> Self {
|
||||
Self {
|
||||
data: MetadataData::import(&path, version),
|
||||
path,
|
||||
data: MetadataData::import(path, version),
|
||||
path: path.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
use std::thread::{self};
|
||||
|
||||
use allocative::Allocative;
|
||||
|
||||
mod _database;
|
||||
mod _trait;
|
||||
mod address_index_to_address_data;
|
||||
mod address_index_to_empty_address_data;
|
||||
@@ -12,24 +9,20 @@ mod txid_to_tx_data;
|
||||
mod txout_index_to_address_index;
|
||||
mod txout_index_to_amount;
|
||||
|
||||
pub use _database::*;
|
||||
use _trait::*;
|
||||
pub use address_index_to_address_data::*;
|
||||
pub use address_index_to_empty_address_data::*;
|
||||
pub use address_to_address_index::*;
|
||||
use itertools::Itertools;
|
||||
use log::info;
|
||||
use metadata::*;
|
||||
use rayon::iter::{IntoParallelIterator, ParallelIterator};
|
||||
use snkrj::AnyDatabase;
|
||||
pub use txid_to_tx_data::*;
|
||||
pub use txout_index_to_address_index::*;
|
||||
pub use txout_index_to_amount::*;
|
||||
|
||||
use crate::{
|
||||
log,
|
||||
structs::{Date, Height},
|
||||
utils::time,
|
||||
Exit,
|
||||
};
|
||||
use crate::structs::{Config, Date, Height};
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct Databases {
|
||||
@@ -42,18 +35,20 @@ pub struct Databases {
|
||||
}
|
||||
|
||||
impl Databases {
|
||||
pub fn import() -> Self {
|
||||
let address_index_to_address_data = AddressIndexToAddressData::init();
|
||||
pub fn import(config: &Config) -> Self {
|
||||
let address_index_to_address_data = AddressIndexToAddressData::init(config);
|
||||
|
||||
let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init();
|
||||
let address_index_to_empty_address_data = AddressIndexToEmptyAddressData::init(config);
|
||||
|
||||
let address_to_address_index = AddressToAddressIndex::init();
|
||||
let address_to_address_index = AddressToAddressIndex::init(config);
|
||||
|
||||
let txid_to_tx_data = TxidToTxData::init();
|
||||
let txid_to_tx_data = TxidToTxData::init(config);
|
||||
|
||||
let txout_index_to_address_index = TxoutIndexToAddressIndex::init();
|
||||
let txout_index_to_address_index = TxoutIndexToAddressIndex::init(config);
|
||||
|
||||
let txout_index_to_amount = TxoutIndexToAmount::init();
|
||||
let txout_index_to_amount = TxoutIndexToAmount::init(config);
|
||||
|
||||
info!("Imported databases");
|
||||
|
||||
Self {
|
||||
address_index_to_address_data,
|
||||
@@ -91,62 +86,21 @@ impl Databases {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn export(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
pub fn export(
|
||||
&mut self,
|
||||
height: Height,
|
||||
date: Date,
|
||||
defragment: bool,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.export_metadata(height, date)?;
|
||||
|
||||
self.drain_to_vec()
|
||||
.into_par_iter()
|
||||
.try_for_each(AnyDatabase::boxed_export)?;
|
||||
.try_for_each(|s| AnyDatabase::boxed_export(s, defragment))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
thread::scope(|s| {
|
||||
s.spawn(|| {
|
||||
self.address_index_to_address_data.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.address_index_to_empty_address_data.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.address_to_address_index.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.txid_to_tx_data.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.txout_index_to_address_index.open_all();
|
||||
});
|
||||
|
||||
s.spawn(|| {
|
||||
self.txout_index_to_amount.open_all();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
pub fn defragment(&mut self, exit: &Exit) {
|
||||
exit.block();
|
||||
|
||||
log("Databases defragmentation");
|
||||
|
||||
time("Defragmenting databases", || {
|
||||
time("Opened all databases", || self.open_all());
|
||||
|
||||
log("Defragmenting...");
|
||||
|
||||
self.drain_to_vec()
|
||||
.into_par_iter()
|
||||
.for_each(AnyDatabase::boxed_defragment);
|
||||
});
|
||||
|
||||
exit.unblock();
|
||||
}
|
||||
|
||||
pub fn reset(&mut self, include_addresses: bool) {
|
||||
if include_addresses {
|
||||
let _ = self.address_index_to_address_data.reset();
|
||||
@@ -177,8 +131,8 @@ impl Databases {
|
||||
|
||||
pub fn check_if_usable(
|
||||
&self,
|
||||
min_initial_last_address_height: Option<Height>,
|
||||
min_initial_last_address_date: Option<Date>,
|
||||
last_address_height: Option<Height>,
|
||||
last_address_date: Option<Date>,
|
||||
) -> bool {
|
||||
let are_tx_databases_in_sync = self
|
||||
.txout_index_to_amount
|
||||
@@ -215,8 +169,7 @@ impl Databases {
|
||||
return false;
|
||||
}
|
||||
|
||||
// let are_address_datasets_farer_or_in_sync_with_address_databases =
|
||||
min_initial_last_address_height >= self.address_to_address_index.metadata.last_height
|
||||
&& min_initial_last_address_date >= self.address_to_address_index.metadata.last_date
|
||||
last_address_height >= self.address_to_address_index.metadata.last_height
|
||||
&& last_address_date >= self.address_to_address_index.metadata.last_date
|
||||
}
|
||||
}
|
||||
+30
-60
@@ -1,16 +1,17 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use biter::bitcoin::Txid;
|
||||
use brk_parser::bitcoin::Txid;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Date, Height, TxData, U8x31};
|
||||
use crate::structs::{Config, TxData, U8x31};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = U8x31;
|
||||
type Value = TxData;
|
||||
@@ -18,25 +19,12 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct TxidToTxData {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<u16, Database>,
|
||||
}
|
||||
|
||||
impl Deref for TxidToTxData {
|
||||
type Target = BTreeMap<u16, Database>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.map
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for TxidToTxData {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.map
|
||||
}
|
||||
}
|
||||
|
||||
impl TxidToTxData {
|
||||
pub fn insert(&mut self, txid: &Txid, tx_index: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
@@ -46,53 +34,38 @@ impl TxidToTxData {
|
||||
self.open_db(txid).insert(txid_key, tx_index)
|
||||
}
|
||||
|
||||
// pub fn safe_get(&mut self, txid: &Txid) -> Option<&Value> {
|
||||
// let txid_key = Self::txid_to_key(txid);
|
||||
// self.open_db(txid).get(&txid_key)
|
||||
// }
|
||||
|
||||
/// Doesn't check if the database is open contrary to `safe_get` which does and opens if needed.
|
||||
/// Though it makes it easy to use with rayon
|
||||
pub fn unsafe_get(&self, txid: &Txid) -> Option<&Value> {
|
||||
pub fn get(&self, txid: &Txid) -> Option<&Value> {
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
let db_index = Self::db_index(txid);
|
||||
|
||||
self.get(&db_index).unwrap().get(&txid_key)
|
||||
self.map.get(&db_index).unwrap().get(&txid_key)
|
||||
}
|
||||
|
||||
// pub fn unsafe_get_from_puts(&self, txid: &Txid) -> Option<&Value> {
|
||||
// let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
// let db_index = Self::db_index(txid);
|
||||
|
||||
// self.get(&db_index).unwrap().get_from_puts(&txid_key)
|
||||
// }
|
||||
|
||||
pub fn unsafe_get_mut_from_puts(&mut self, txid: &Txid) -> Option<&mut Value> {
|
||||
pub fn get_mut_from_ram(&mut self, txid: &Txid) -> Option<&mut Value> {
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
let db_index = Self::db_index(txid);
|
||||
|
||||
self.get_mut(&db_index)
|
||||
.unwrap()
|
||||
.get_mut_from_puts(&txid_key)
|
||||
self.map.get_mut(&db_index).unwrap().get_mut_from_ram(&txid_key)
|
||||
}
|
||||
|
||||
pub fn remove_from_db(&mut self, txid: &Txid) {
|
||||
pub fn remove_later_from_disk(&mut self, txid: &Txid) {
|
||||
self.metadata.called_remove();
|
||||
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
self.open_db(txid).db_remove(&txid_key);
|
||||
self.open_db(txid).remove_later_from_disk(&txid_key);
|
||||
}
|
||||
|
||||
pub fn remove_from_puts(&mut self, txid: &Txid) {
|
||||
pub fn remove_from_ram(&mut self, txid: &Txid) {
|
||||
self.metadata.called_remove();
|
||||
|
||||
let txid_key = Self::txid_to_key(txid);
|
||||
|
||||
self.open_db(txid).remove_from_puts(&txid_key);
|
||||
self.open_db(txid).remove_from_ram(&txid_key);
|
||||
}
|
||||
|
||||
pub fn update(&mut self, txid: &Txid, tx_data: TxData) {
|
||||
@@ -108,9 +81,10 @@ impl TxidToTxData {
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn _open_db(&mut self, db_index: u16) -> &mut Database {
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let path = Self::root().join(db_index.to_string());
|
||||
fn _open_db(&mut self, db_index: u16) -> &mut Database {
|
||||
let path = self.path.to_owned();
|
||||
self.map.entry(db_index).or_insert_with(|| {
|
||||
let path = path.join(db_index.to_string());
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
}
|
||||
@@ -125,31 +99,23 @@ impl TxidToTxData {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for TxidToTxData {
|
||||
fn import() -> Self {
|
||||
let metadata = Metadata::import(Self::root(), 2);
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("txid_to_tx_data");
|
||||
let metadata = Metadata::import(&path, 2);
|
||||
|
||||
Self {
|
||||
path,
|
||||
metadata,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"txid_to_tx_data"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -180,7 +146,11 @@ impl AnyDatabaseGroup for TxidToTxData {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+21
-30
@@ -2,14 +2,16 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Date, Height, TxoutIndex};
|
||||
use crate::structs::{Config, TxoutIndex};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = TxoutIndex;
|
||||
type Value = u32;
|
||||
@@ -17,8 +19,9 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct TxoutIndexToAddressIndex {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -39,20 +42,12 @@ impl DerefMut for TxoutIndexToAddressIndex {
|
||||
const DB_MAX_SIZE: usize = 10_000_000_000;
|
||||
|
||||
impl TxoutIndexToAddressIndex {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
|
||||
// self.open_db(key).remove_from_puts(key).map(|v| {
|
||||
// self.metadata.called_remove();
|
||||
|
||||
// v
|
||||
// })
|
||||
// }
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.metadata.called_remove();
|
||||
|
||||
@@ -69,6 +64,7 @@ impl TxoutIndexToAddressIndex {
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path.to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -77,7 +73,7 @@ impl TxoutIndexToAddressIndex {
|
||||
(db_index + 1) * DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
@@ -89,30 +85,21 @@ impl TxoutIndexToAddressIndex {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("txout_index_to_address_index");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"txout_index_to_address_index"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -151,7 +138,11 @@ impl AnyDatabaseGroup for TxoutIndexToAddressIndex {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+21
-30
@@ -2,14 +2,16 @@ use std::{
|
||||
collections::BTreeMap,
|
||||
fs, mem,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use allocative::Allocative;
|
||||
use itertools::Itertools;
|
||||
use snkrj::{AnyDatabase, Database as _Database};
|
||||
|
||||
use crate::structs::{Amount, Date, Height, TxoutIndex};
|
||||
use crate::structs::{Amount, Config, TxoutIndex};
|
||||
|
||||
use super::{AnyDatabase, AnyDatabaseGroup, Database as _Database, Metadata};
|
||||
use super::{AnyDatabaseGroup, Metadata};
|
||||
|
||||
type Key = TxoutIndex;
|
||||
type Value = Amount;
|
||||
@@ -17,8 +19,9 @@ type Database = _Database<Key, Value>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct TxoutIndexToAmount {
|
||||
path: PathBuf,
|
||||
pub metadata: Metadata,
|
||||
|
||||
#[allocative(skip)]
|
||||
map: BTreeMap<usize, Database>,
|
||||
}
|
||||
|
||||
@@ -39,20 +42,12 @@ impl DerefMut for TxoutIndexToAmount {
|
||||
const DB_MAX_SIZE: usize = 10_000_000_000;
|
||||
|
||||
impl TxoutIndexToAmount {
|
||||
pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.metadata.called_insert();
|
||||
|
||||
self.open_db(&key).unsafe_insert(key, value)
|
||||
self.open_db(&key).insert_to_ram(key, value)
|
||||
}
|
||||
|
||||
// pub fn undo_insert(&mut self, key: &Key) -> Option<Value> {
|
||||
// self.open_db(key).remove_from_puts(key).map(|v| {
|
||||
// self.metadata.called_remove();
|
||||
|
||||
// v
|
||||
// })
|
||||
// }
|
||||
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.metadata.called_remove();
|
||||
|
||||
@@ -69,6 +64,7 @@ impl TxoutIndexToAmount {
|
||||
|
||||
pub fn open_db(&mut self, key: &Key) -> &mut Database {
|
||||
let db_index = Self::db_index(key);
|
||||
let path = self.path.to_owned();
|
||||
|
||||
self.entry(db_index).or_insert_with(|| {
|
||||
let db_name = format!(
|
||||
@@ -77,7 +73,7 @@ impl TxoutIndexToAmount {
|
||||
(db_index + 1) * DB_MAX_SIZE
|
||||
);
|
||||
|
||||
let path = Self::root().join(db_name);
|
||||
let path = path.join(db_name);
|
||||
|
||||
Database::open(path).unwrap()
|
||||
})
|
||||
@@ -89,30 +85,21 @@ impl TxoutIndexToAmount {
|
||||
}
|
||||
|
||||
impl AnyDatabaseGroup for TxoutIndexToAmount {
|
||||
fn import() -> Self {
|
||||
fn import(config: &Config) -> Self {
|
||||
let path = config.path_databases().join("txout_index_to_amount");
|
||||
Self {
|
||||
metadata: Metadata::import(Self::root(), 1),
|
||||
|
||||
metadata: Metadata::import(&path, 1),
|
||||
path,
|
||||
map: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_dir_all(&self) -> color_eyre::Result<(), std::io::Error> {
|
||||
fs::create_dir_all(Self::root())
|
||||
}
|
||||
|
||||
fn reset_metadata(&mut self) {
|
||||
self.metadata.reset();
|
||||
}
|
||||
|
||||
fn folder<'a>() -> &'a str {
|
||||
"txout_index_to_amount"
|
||||
}
|
||||
|
||||
fn open_all(&mut self) {
|
||||
let path = Self::root();
|
||||
|
||||
let folder = fs::read_dir(path);
|
||||
let folder = fs::read_dir(&self.path);
|
||||
|
||||
if folder.is_err() {
|
||||
return;
|
||||
@@ -151,7 +138,11 @@ impl AnyDatabaseGroup for TxoutIndexToAmount {
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn export_metadata(&mut self, height: Height, date: Date) -> color_eyre::Result<()> {
|
||||
self.metadata.export(height, date)
|
||||
fn metadata(&mut self) -> &mut Metadata {
|
||||
&mut self.metadata
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -3,14 +3,14 @@ use rayon::prelude::*;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{
|
||||
parser::datasets::{
|
||||
cohort_metadata::AddressCohortMetadataDataset, ComputeData, DateRecapDataset, RatioDataset,
|
||||
SubDataset,
|
||||
},
|
||||
structs::{
|
||||
AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, BiMap, Date, Height, MapKind, Timestamp, OHLC,
|
||||
AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, BiMap, Date, DateMap, Height, HeightMap,
|
||||
MapKind, Timestamp, OHLC,
|
||||
},
|
||||
DateMap, HeightMap,
|
||||
};
|
||||
|
||||
use super::{AnyDatasetGroup, MinInitialStates};
|
||||
+4
-7
@@ -29,6 +29,10 @@ impl MinInitialStates {
|
||||
computed: MinInitialState::compute_from_datasets(datasets, Mode::Computed, config),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn min_last_height(&self) -> Option<Height> {
|
||||
self.computed.last_height.min(self.inserted.last_height)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Copy, Allocative)]
|
||||
@@ -45,13 +49,6 @@ enum Mode {
|
||||
}
|
||||
|
||||
impl MinInitialState {
|
||||
// pub fn consume(&mut self, other: Self) {
|
||||
// self.first_unsafe_date = other.first_unsafe_date;
|
||||
// self.first_unsafe_height = other.first_unsafe_height;
|
||||
// self.last_date = other.last_date;
|
||||
// self.last_height = other.last_height;
|
||||
// }
|
||||
|
||||
fn compute_from_datasets(datasets: &dyn AnyDatasets, mode: Mode, config: &Config) -> Self {
|
||||
match mode {
|
||||
Mode::Inserted => {
|
||||
+4
-4
@@ -2,8 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -15,8 +15,8 @@ pub struct AllAddressesMetadataDataset {
|
||||
}
|
||||
|
||||
impl AllAddressesMetadataDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(path: &MapPath, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| path.join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -3,9 +3,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::{AddressCohortId, DurableStates},
|
||||
structs::{AddressSplit, BiMap, Config, Date, Height},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::{AddressCohortId, DurableStates},
|
||||
},
|
||||
structs::{AddressSplit, BiMap, Config, Date, Height, MapPath},
|
||||
};
|
||||
|
||||
use super::cohort_metadata::AddressCohortMetadataDataset;
|
||||
@@ -23,7 +25,7 @@ pub struct CohortDataset {
|
||||
|
||||
impl CohortDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
id: AddressCohortId,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -33,8 +35,8 @@ impl CohortDataset {
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
split,
|
||||
metadata: AddressCohortMetadataDataset::import(parent_path, &name, config)?,
|
||||
subs: SubDataset::import(parent_path, &name, config)?,
|
||||
metadata: AddressCohortMetadataDataset::import(path, &name, config)?,
|
||||
subs: SubDataset::import(path, &name, config)?,
|
||||
};
|
||||
|
||||
s.min_initial_states
|
||||
+5
-5
@@ -2,8 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -19,15 +19,15 @@ pub struct AddressCohortMetadataDataset {
|
||||
|
||||
impl AddressCohortMetadataDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -7,9 +7,8 @@ use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
states::SplitByAddressCohort,
|
||||
structs::{BiMap, Config, Height},
|
||||
Date,
|
||||
parser::states::SplitByAddressCohort,
|
||||
structs::{BiMap, Config, Date, Height},
|
||||
};
|
||||
|
||||
use self::{all_metadata::AllAddressesMetadataDataset, cohort::CohortDataset};
|
||||
@@ -26,13 +25,15 @@ pub struct AddressDatasets {
|
||||
}
|
||||
|
||||
impl AddressDatasets {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let mut cohorts = SplitByAddressCohort::<Option<CohortDataset>>::default();
|
||||
|
||||
let path_dataset = config.path_datasets();
|
||||
|
||||
cohorts
|
||||
.as_vec()
|
||||
.into_par_iter()
|
||||
.map(|(_, id)| (id, CohortDataset::import(parent_path, id, config)))
|
||||
.map(|(_, id)| (id, CohortDataset::import(&path_dataset, id, config)))
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
|
||||
@@ -43,7 +44,7 @@ impl AddressDatasets {
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
|
||||
metadata: AllAddressesMetadataDataset::import(parent_path, config)?,
|
||||
metadata: AllAddressesMetadataDataset::import(&path_dataset, config)?,
|
||||
|
||||
cohorts: cohorts.unwrap(),
|
||||
};
|
||||
@@ -2,7 +2,7 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, Date, HeightMap, MapKind, Timestamp},
|
||||
};
|
||||
|
||||
@@ -16,8 +16,8 @@ pub struct BlockMetadataDataset {
|
||||
}
|
||||
|
||||
impl BlockMetadataDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,9 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
structs::{Config, MapKind},
|
||||
DateMap, HeightMap,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, DateMap, HeightMap, MapKind},
|
||||
};
|
||||
|
||||
use super::{InsertData, MinInitialStates};
|
||||
@@ -17,8 +16,8 @@ pub struct CoindaysDataset {
|
||||
}
|
||||
|
||||
impl CoindaysDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,9 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
structs::{BiMap, Config, DateMap, Height, MapKind},
|
||||
structs::{BiMap, Config, DateMap, Height, HeightMap, MapKind},
|
||||
utils::{ONE_DAY_IN_DAYS, ONE_YEAR_IN_DAYS, THREE_MONTHS_IN_DAYS, TWO_WEEK_IN_DAYS},
|
||||
HeightMap,
|
||||
};
|
||||
|
||||
use super::{AnyDataset, ComputeData, InsertData, MinInitialStates, RatioDataset};
|
||||
@@ -72,8 +71,9 @@ pub struct CointimeDataset {
|
||||
}
|
||||
|
||||
impl CointimeDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let path_dataset = config.path_datasets();
|
||||
let f = |s: &str| path_dataset.join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -93,7 +93,7 @@ impl CointimeDataset {
|
||||
// Computed
|
||||
active_cap: BiMap::new_bin(1, MapKind::Computed, &f("active_cap")),
|
||||
active_price: BiMap::new_bin(1, MapKind::Computed, &f("active_price")),
|
||||
active_price_ratio: RatioDataset::import(parent_path, "active_price", config)?,
|
||||
active_price_ratio: RatioDataset::import(&path_dataset, "active_price", config)?,
|
||||
active_supply: BiMap::new_bin(1, MapKind::Computed, &f("active_supply")),
|
||||
active_supply_3m_net_change: BiMap::new_bin(
|
||||
1,
|
||||
@@ -140,7 +140,7 @@ impl CointimeDataset {
|
||||
),
|
||||
cointime_cap: BiMap::new_bin(1, MapKind::Computed, &f("cointime_cap")),
|
||||
cointime_price: BiMap::new_bin(1, MapKind::Computed, &f("cointime_price")),
|
||||
cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price", config)?,
|
||||
cointime_price_ratio: RatioDataset::import(&path_dataset, "cointime_price", config)?,
|
||||
cointime_value_created: HeightMap::new_bin(
|
||||
1,
|
||||
MapKind::Computed,
|
||||
@@ -237,7 +237,11 @@ impl CointimeDataset {
|
||||
&f("true_market_deviation"),
|
||||
),
|
||||
true_market_mean: BiMap::new_bin(1, MapKind::Computed, &f("true_market_mean")),
|
||||
true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean", config)?,
|
||||
true_market_mean_ratio: RatioDataset::import(
|
||||
&path_dataset,
|
||||
"true_market_mean",
|
||||
config,
|
||||
)?,
|
||||
true_market_net_unrealized_profit_and_loss: BiMap::new_bin(
|
||||
1,
|
||||
MapKind::Computed,
|
||||
@@ -245,7 +249,7 @@ impl CointimeDataset {
|
||||
),
|
||||
vaulted_cap: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_cap")),
|
||||
vaulted_price: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_price")),
|
||||
vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price", config)?,
|
||||
vaulted_price_ratio: RatioDataset::import(&path_dataset, "vaulted_price", config)?,
|
||||
vaulted_supply: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_supply")),
|
||||
vaulted_supply_3m_net_change: BiMap::new_bin(
|
||||
1,
|
||||
@@ -16,8 +16,8 @@ pub struct ConstantDataset {
|
||||
}
|
||||
|
||||
impl ConstantDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,7 +2,7 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, DateMap, Height, MapKind},
|
||||
};
|
||||
|
||||
@@ -17,8 +17,8 @@ pub struct DateMetadataDataset {
|
||||
}
|
||||
|
||||
impl DateMetadataDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -4,7 +4,7 @@ use ordered_float::OrderedFloat;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDataset,
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Amount, BiMap, Config, DateMap, Height, HeightMap, MapKey, MapKind},
|
||||
utils::{
|
||||
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
|
||||
@@ -123,8 +123,8 @@ pub struct MiningDataset {
|
||||
}
|
||||
|
||||
impl MiningDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -1,9 +1,10 @@
|
||||
use std::{collections::BTreeMap, fs, ops::RangeInclusive, path::Path};
|
||||
use std::{collections::BTreeMap, ops::RangeInclusive};
|
||||
|
||||
use allocative::Allocative;
|
||||
|
||||
use itertools::Itertools;
|
||||
|
||||
use log::info;
|
||||
use rayon::prelude::*;
|
||||
|
||||
mod _traits;
|
||||
@@ -34,16 +35,18 @@ pub use transaction::*;
|
||||
pub use utxo::*;
|
||||
|
||||
use crate::{
|
||||
databases::Databases,
|
||||
io::{Json, JSON_EXTENSION},
|
||||
states::{
|
||||
AddressCohortsInputStates,
|
||||
AddressCohortsOneShotStates,
|
||||
AddressCohortsRealizedStates,
|
||||
States,
|
||||
UTXOCohortsOneShotStates,
|
||||
// UTXOCohortsReceivedStates,
|
||||
UTXOCohortsSentStates,
|
||||
io::Json,
|
||||
parser::{
|
||||
databases::Databases,
|
||||
states::{
|
||||
AddressCohortsInputStates,
|
||||
AddressCohortsOneShotStates,
|
||||
AddressCohortsRealizedStates,
|
||||
States,
|
||||
UTXOCohortsOneShotStates,
|
||||
// UTXOCohortsReceivedStates,
|
||||
UTXOCohortsSentStates,
|
||||
},
|
||||
},
|
||||
structs::{Amount, Config, Date, Height, Price, Timestamp},
|
||||
};
|
||||
@@ -84,7 +87,7 @@ pub struct ComputeData<'a> {
|
||||
}
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct AllDatasets {
|
||||
pub struct Datasets {
|
||||
min_initial_states: MinInitialStates,
|
||||
|
||||
pub constant: ConstantDataset,
|
||||
@@ -99,31 +102,27 @@ pub struct AllDatasets {
|
||||
pub utxo: UTXODatasets,
|
||||
}
|
||||
|
||||
const DATASETS_PATH: &str = "../datasets";
|
||||
|
||||
impl AllDatasets {
|
||||
impl Datasets {
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let path = DATASETS_PATH;
|
||||
let price = PriceDatasets::import(config)?;
|
||||
|
||||
let price = PriceDatasets::import(path, config)?;
|
||||
let constant = ConstantDataset::import(config)?;
|
||||
|
||||
let constant = ConstantDataset::import(path, config)?;
|
||||
let date_metadata = DateMetadataDataset::import(config)?;
|
||||
|
||||
let date_metadata = DateMetadataDataset::import(path, config)?;
|
||||
let cointime = CointimeDataset::import(config)?;
|
||||
|
||||
let cointime = CointimeDataset::import(path, config)?;
|
||||
let coindays = CoindaysDataset::import(config)?;
|
||||
|
||||
let coindays = CoindaysDataset::import(path, config)?;
|
||||
let mining = MiningDataset::import(config)?;
|
||||
|
||||
let mining = MiningDataset::import(path, config)?;
|
||||
let block_metadata = BlockMetadataDataset::import(config)?;
|
||||
|
||||
let block_metadata = BlockMetadataDataset::import(path, config)?;
|
||||
let transaction = TransactionDataset::import(config)?;
|
||||
|
||||
let transaction = TransactionDataset::import(path, config)?;
|
||||
let address = AddressDatasets::import(config)?;
|
||||
|
||||
let address = AddressDatasets::import(path, config)?;
|
||||
|
||||
let utxo = UTXODatasets::import(path, config)?;
|
||||
let utxo = UTXODatasets::import(config)?;
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -140,14 +139,18 @@ impl AllDatasets {
|
||||
utxo,
|
||||
};
|
||||
|
||||
s.min_initial_states
|
||||
.consume(MinInitialStates::compute_from_datasets(&s, config));
|
||||
s.set_initial_states(config);
|
||||
|
||||
s.export_meta_files()?;
|
||||
info!("Imported datasets");
|
||||
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
fn set_initial_states(&mut self, config: &Config) {
|
||||
self.min_initial_states
|
||||
.consume(MinInitialStates::compute_from_datasets(self, config));
|
||||
}
|
||||
|
||||
pub fn insert(&mut self, insert_data: InsertData) {
|
||||
if insert_data.compute_addresses {
|
||||
self.address.insert(&insert_data);
|
||||
@@ -229,21 +232,6 @@ impl AllDatasets {
|
||||
&mut self.price.market_cap,
|
||||
);
|
||||
|
||||
// No compute needed for now
|
||||
// if self.block_metadata.should_compute(height, date) {
|
||||
// self.block_metadata.compute(&compute_data);
|
||||
// }
|
||||
|
||||
// No compute needed for now
|
||||
// if self.date_metadata.should_compute(height, date) {
|
||||
// self.date_metadata.compute(&compute_data);
|
||||
// }
|
||||
|
||||
// No compute needed for now
|
||||
// if self.coindays.should_compute(height, date) {
|
||||
// self.coindays.compute(&compute_data);
|
||||
// }
|
||||
|
||||
if self.transaction.should_compute(&compute_data) {
|
||||
self.transaction.compute(
|
||||
&compute_data,
|
||||
@@ -269,43 +257,12 @@ impl AllDatasets {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn export_meta_files(&self) -> color_eyre::Result<()> {
|
||||
let mut path_to_type: BTreeMap<&Path, &str> = self
|
||||
.to_any_dataset_vec()
|
||||
.into_iter()
|
||||
.flat_map(|dataset| dataset.to_all_map_vec())
|
||||
.flat_map(|map| map.exported_path_with_t_name())
|
||||
.collect();
|
||||
pub fn export(&mut self, config: &Config, height: Height) -> color_eyre::Result<()> {
|
||||
let is_new = self
|
||||
.min_initial_states
|
||||
.min_last_height()
|
||||
.map_or(true, |last| last <= height);
|
||||
|
||||
path_to_type.insert(Path::new("../datasets/last"), "Value");
|
||||
|
||||
let datasets_len = path_to_type.len();
|
||||
|
||||
let server_inputs_path = "../server/in";
|
||||
|
||||
fs::create_dir_all(server_inputs_path)?;
|
||||
|
||||
Json::export(
|
||||
Path::new(&format!("{server_inputs_path}/disk_path_to_type.json")),
|
||||
&path_to_type,
|
||||
)?;
|
||||
|
||||
let datasets_len_path = format!("{server_inputs_path}/datasets_len.txt");
|
||||
|
||||
if let Ok(len) = fs::read_to_string(&datasets_len_path) {
|
||||
if let Ok(len) = len.parse::<usize>() {
|
||||
if datasets_len == len {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs::write(datasets_len_path, datasets_len.to_string())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn export(&mut self) -> color_eyre::Result<()> {
|
||||
self.to_mut_any_dataset_vec()
|
||||
.into_iter()
|
||||
.for_each(|dataset| dataset.pre_export());
|
||||
@@ -321,39 +278,28 @@ impl AllDatasets {
|
||||
.for_each(|dataset| {
|
||||
dataset.post_export();
|
||||
|
||||
dataset.to_all_map_vec().iter().for_each(|map| {
|
||||
if let Some(last_path) = map.path_last() {
|
||||
if let Some(last_value) = map.last_value() {
|
||||
let mut last_path = last_path.clone();
|
||||
last_path.pop();
|
||||
|
||||
let last_path = last_path.to_str().unwrap();
|
||||
|
||||
let skip = if last_path.starts_with(DATASETS_PATH) {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
|
||||
path_to_last.insert(
|
||||
last_path.split('/').skip(skip).join("-").to_string(),
|
||||
last_value,
|
||||
);
|
||||
if is_new {
|
||||
dataset.to_all_map_vec().iter().for_each(|map| {
|
||||
if map.path_last().is_some() {
|
||||
if let Some(last_value) = map.last_value() {
|
||||
path_to_last.insert(map.id(config), last_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Json::export(
|
||||
Path::new(&format!("{DATASETS_PATH}/last.{JSON_EXTENSION}")),
|
||||
&path_to_last,
|
||||
)?;
|
||||
if is_new {
|
||||
Json::export(&config.path_datasets_last_values(), &path_to_last)?;
|
||||
}
|
||||
|
||||
self.set_initial_states(config);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl AnyDatasets for AllDatasets {
|
||||
impl AnyDatasets for Datasets {
|
||||
fn get_min_initial_states(&self) -> &MinInitialStates {
|
||||
&self.min_initial_states
|
||||
}
|
||||
@@ -7,7 +7,7 @@ use color_eyre::eyre::Error;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
price::{Binance, Kibo, Kraken},
|
||||
parser::price::{Binance, Kibo, Kraken},
|
||||
structs::{
|
||||
Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey,
|
||||
MapKind, Timestamp, OHLC,
|
||||
@@ -85,10 +85,9 @@ pub struct PriceDatasets {
|
||||
}
|
||||
|
||||
impl PriceDatasets {
|
||||
pub fn import(datasets_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let price_path = "../price";
|
||||
|
||||
let f = |s: &str| format!("{datasets_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let path_dataset = config.path_datasets();
|
||||
let f = |s: &str| path_dataset.join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -104,7 +103,7 @@ impl PriceDatasets {
|
||||
// ---
|
||||
// Inserted
|
||||
// ---
|
||||
ohlc: BiMap::new_json(1, MapKind::Inserted, price_path),
|
||||
ohlc: BiMap::new_json(1, MapKind::Inserted, &config.path_price()),
|
||||
|
||||
// ---
|
||||
// Computed
|
||||
@@ -115,31 +114,31 @@ impl PriceDatasets {
|
||||
close: BiMap::new_bin(1, MapKind::Computed, &f("close")),
|
||||
market_cap: BiMap::new_bin(1, MapKind::Computed, &f("market_cap")),
|
||||
price_1w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1w_sma")),
|
||||
price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma", config)?,
|
||||
price_1w_sma_ratio: RatioDataset::import(&path_dataset, "price_1w_sma", config)?,
|
||||
price_1m_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1m_sma")),
|
||||
price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma", config)?,
|
||||
price_1m_sma_ratio: RatioDataset::import(&path_dataset, "price_1m_sma", config)?,
|
||||
price_1y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1y_sma")),
|
||||
price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma", config)?,
|
||||
price_1y_sma_ratio: RatioDataset::import(&path_dataset, "price_1y_sma", config)?,
|
||||
price_2y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_2y_sma")),
|
||||
price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma", config)?,
|
||||
price_2y_sma_ratio: RatioDataset::import(&path_dataset, "price_2y_sma", config)?,
|
||||
price_4y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_4y_sma")),
|
||||
price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma", config)?,
|
||||
price_4y_sma_ratio: RatioDataset::import(&path_dataset, "price_4y_sma", config)?,
|
||||
price_8d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_8d_sma")),
|
||||
price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma", config)?,
|
||||
price_8d_sma_ratio: RatioDataset::import(&path_dataset, "price_8d_sma", config)?,
|
||||
price_13d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_13d_sma")),
|
||||
price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma", config)?,
|
||||
price_13d_sma_ratio: RatioDataset::import(&path_dataset, "price_13d_sma", config)?,
|
||||
price_21d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_21d_sma")),
|
||||
price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma", config)?,
|
||||
price_21d_sma_ratio: RatioDataset::import(&path_dataset, "price_21d_sma", config)?,
|
||||
price_34d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_34d_sma")),
|
||||
price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma", config)?,
|
||||
price_34d_sma_ratio: RatioDataset::import(&path_dataset, "price_34d_sma", config)?,
|
||||
price_55d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_55d_sma")),
|
||||
price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma", config)?,
|
||||
price_55d_sma_ratio: RatioDataset::import(&path_dataset, "price_55d_sma", config)?,
|
||||
price_89d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_89d_sma")),
|
||||
price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma", config)?,
|
||||
price_89d_sma_ratio: RatioDataset::import(&path_dataset, "price_89d_sma", config)?,
|
||||
price_144d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_144d_sma")),
|
||||
price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma", config)?,
|
||||
price_144d_sma_ratio: RatioDataset::import(&path_dataset, "price_144d_sma", config)?,
|
||||
price_200w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_200w_sma")),
|
||||
price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma", config)?,
|
||||
price_200w_sma_ratio: RatioDataset::import(&path_dataset, "price_200w_sma", config)?,
|
||||
price_1d_total_return: DateMap::new_bin(
|
||||
1,
|
||||
MapKind::Computed,
|
||||
@@ -540,6 +539,7 @@ impl PriceDatasets {
|
||||
height: Height,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if let Some(ohlc) = self.ohlc.height.get_or_import(&height) {
|
||||
return Ok(ohlc);
|
||||
@@ -558,7 +558,7 @@ impl PriceDatasets {
|
||||
.unwrap_or_else(|_| {
|
||||
self.get_from_1mn_binance(timestamp, previous_timestamp)
|
||||
.unwrap_or_else(|_| {
|
||||
self.get_from_har_binance(timestamp, previous_timestamp)
|
||||
self.get_from_har_binance(timestamp, previous_timestamp, config)
|
||||
.unwrap_or_else(|_| {
|
||||
self.get_from_height_kibo(&height).unwrap_or_else(|_| {
|
||||
let date = timestamp.to_date();
|
||||
@@ -659,10 +659,11 @@ How to fix this:
|
||||
&mut self,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<OHLC> {
|
||||
if self.binance_har.is_none() {
|
||||
self.binance_har
|
||||
.replace(Binance::read_har_file().unwrap_or_default());
|
||||
.replace(Binance::read_har_file(config).unwrap_or_default());
|
||||
}
|
||||
|
||||
Self::find_height_ohlc(
|
||||
+9
-7
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::CapitalizationState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::CapitalizationState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
utils::ONE_MONTH_IN_DAYS,
|
||||
};
|
||||
|
||||
@@ -22,15 +24,15 @@ pub struct CapitalizationDataset {
|
||||
|
||||
impl CapitalizationDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -52,7 +54,7 @@ impl CapitalizationDataset {
|
||||
),
|
||||
realized_price: BiMap::new_bin(1, MapKind::Computed, &f("realized_price")),
|
||||
realized_price_ratio: RatioDataset::import(
|
||||
parent_path,
|
||||
path,
|
||||
&format!(
|
||||
"{}realized_price",
|
||||
name.as_ref().map_or("".to_owned(), |n| format!("{n}-"))
|
||||
@@ -2,10 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::InputState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
DateMap, HeightMap,
|
||||
parser::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::InputState,
|
||||
},
|
||||
structs::{BiMap, Config, DateMap, HeightMap, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -22,15 +23,15 @@ pub struct InputSubDataset {
|
||||
|
||||
impl InputSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -21,7 +21,10 @@ pub use supply::*;
|
||||
pub use unrealized::*;
|
||||
pub use utxo::*;
|
||||
|
||||
use crate::{datasets::AnyDataset, structs::Config};
|
||||
use crate::{
|
||||
parser::datasets::AnyDataset,
|
||||
structs::{Config, MapPath},
|
||||
};
|
||||
|
||||
use super::AnyDatasetGroup;
|
||||
|
||||
@@ -39,7 +42,7 @@ pub struct SubDataset {
|
||||
|
||||
impl SubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
parent_path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::PricePaidState,
|
||||
structs::{BiMap, Config, Date, Height, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::PricePaidState,
|
||||
},
|
||||
structs::{BiMap, Config, Date, Height, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -34,15 +36,15 @@ pub struct PricePaidSubDataset {
|
||||
|
||||
impl PricePaidSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,8 +2,8 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::datasets::{AnyDataset, ComputeData, MinInitialStates},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
|
||||
};
|
||||
|
||||
@@ -31,9 +31,9 @@ pub struct RatioDataset {
|
||||
}
|
||||
|
||||
impl RatioDataset {
|
||||
pub fn import(parent_path: &str, name: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f_ratio = |s: &str| format!("{parent_path}/market_price_to_{name}_{s}");
|
||||
let f_price = |s: &str| format!("{parent_path}/{name}_{s}");
|
||||
pub fn import(path: &MapPath, name: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f_ratio = |s: &str| path.join(&format!("market_price_to_{name}_{s}"));
|
||||
let f_price = |s: &str| path.join(&format!("{name}_{s}"));
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,11 +2,12 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::RealizedState,
|
||||
structs::{BiMap, Config, MapKind, Price},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::RealizedState,
|
||||
},
|
||||
structs::{BiMap, Config, DateMap, HeightMap, MapKind, MapPath, Price},
|
||||
utils::ONE_MONTH_IN_DAYS,
|
||||
DateMap, HeightMap,
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -45,18 +46,17 @@ pub struct RealizedSubDataset {
|
||||
|
||||
impl RealizedSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
|
||||
@@ -3,12 +3,14 @@ use std::{iter::Sum, ops::Add};
|
||||
use allocative::Allocative;
|
||||
|
||||
use crate::{
|
||||
structs::{DateMapChunkId, GenericMap, MapKey, MapKind, MapSerialized, MapValue},
|
||||
structs::{
|
||||
Date, DateMapChunkId, GenericMap, MapChunkId, MapKey, MapKind, MapPath, MapSerialized,
|
||||
MapValue, SerializedDateMap,
|
||||
},
|
||||
utils::{get_percentile, LossyFrom},
|
||||
Date, MapChunkId, SerializedBTreeMap,
|
||||
};
|
||||
|
||||
pub type DateRecapDataset<T> = RecapDataset<Date, T, DateMapChunkId, SerializedBTreeMap<Date, T>>;
|
||||
pub type DateRecapDataset<T> = RecapDataset<Date, T, DateMapChunkId, SerializedDateMap<T>>;
|
||||
|
||||
#[derive(Allocative)]
|
||||
pub struct RecapDataset<Key, Value, ChunkId, Serialized> {
|
||||
@@ -91,8 +93,8 @@ where
|
||||
Key: MapKey<ChunkId>,
|
||||
Serialized: MapSerialized<Key, Value, ChunkId>,
|
||||
{
|
||||
pub fn import(parent_path: &str, options: RecapOptions) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(path: &MapPath, options: RecapOptions) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| path.join(s);
|
||||
|
||||
let s = Self {
|
||||
// ---
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::SupplyState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::SupplyState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -19,15 +21,15 @@ pub struct SupplySubDataset {
|
||||
|
||||
impl SupplySubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::UnrealizedState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
|
||||
states::UnrealizedState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -26,15 +28,15 @@ pub struct UnrealizedSubDataset {
|
||||
|
||||
impl UnrealizedSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::UTXOState,
|
||||
structs::{BiMap, Config, MapKind},
|
||||
parser::{
|
||||
datasets::{AnyDataset, InsertData, MinInitialStates},
|
||||
states::UTXOState,
|
||||
},
|
||||
structs::{BiMap, Config, MapKind, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -16,15 +18,15 @@ pub struct UTXOSubDataset {
|
||||
|
||||
impl UTXOSubDataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
path: &MapPath,
|
||||
name: &Option<String>,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| {
|
||||
if let Some(name) = name {
|
||||
format!("{parent_path}/{name}/{s}")
|
||||
path.join(&format!("{name}/{s}"))
|
||||
} else {
|
||||
format!("{parent_path}/{s}")
|
||||
path.join(s)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,12 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::InsertData,
|
||||
structs::{BiMap, Config, HeightMap, MapKind},
|
||||
parser::datasets::InsertData,
|
||||
structs::{BiMap, Config, DateMap, HeightMap, MapKind},
|
||||
utils::{
|
||||
ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, TARGET_BLOCKS_PER_DAY,
|
||||
},
|
||||
DateMap,
|
||||
};
|
||||
|
||||
use super::{AnyDataset, ComputeData, MinInitialStates};
|
||||
@@ -52,8 +51,8 @@ pub struct TransactionDataset {
|
||||
}
|
||||
|
||||
impl TransactionDataset {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| format!("{parent_path}/{s}");
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let f = |s: &str| config.path_datasets().join(s);
|
||||
|
||||
let mut s = Self {
|
||||
min_initial_states: MinInitialStates::default(),
|
||||
@@ -2,9 +2,11 @@ use allocative::Allocative;
|
||||
use struct_iterable::Iterable;
|
||||
|
||||
use crate::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::UTXOCohortId,
|
||||
structs::{BiMap, Config, Date, Height},
|
||||
parser::{
|
||||
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
|
||||
states::UTXOCohortId,
|
||||
},
|
||||
structs::{BiMap, Config, Date, Height, MapPath},
|
||||
};
|
||||
|
||||
#[derive(Allocative, Iterable)]
|
||||
@@ -18,7 +20,7 @@ pub struct UTXODataset {
|
||||
|
||||
impl UTXODataset {
|
||||
pub fn import(
|
||||
parent_path: &str,
|
||||
parent_path: &MapPath,
|
||||
id: UTXOCohortId,
|
||||
config: &Config,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -7,8 +7,8 @@ use rayon::prelude::*;
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::{
|
||||
datasets::AnyDatasets,
|
||||
states::{SplitByUTXOCohort, UTXOCohortId},
|
||||
parser::datasets::AnyDatasets,
|
||||
parser::states::{SplitByUTXOCohort, UTXOCohortId},
|
||||
structs::{BiMap, Config, Date, Height},
|
||||
};
|
||||
|
||||
@@ -22,13 +22,15 @@ pub struct UTXODatasets {
|
||||
}
|
||||
|
||||
impl UTXODatasets {
|
||||
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
|
||||
pub fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
let mut cohorts = SplitByUTXOCohort::<Option<UTXODataset>>::default();
|
||||
|
||||
let path_dataset = config.path_datasets();
|
||||
|
||||
cohorts
|
||||
.as_vec()
|
||||
.into_par_iter()
|
||||
.map(|(_, id)| (id, UTXODataset::import(parent_path, id, config)))
|
||||
.map(|(_, id)| (id, UTXODataset::import(&path_dataset, id, config)))
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
|
||||
@@ -0,0 +1,42 @@
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use brk_parser::bitcoincore_rpc::{Client, RpcApi};
|
||||
|
||||
mod actions;
|
||||
mod databases;
|
||||
mod datasets;
|
||||
mod price;
|
||||
mod states;
|
||||
|
||||
pub use actions::*;
|
||||
pub use databases::*;
|
||||
pub use datasets::*;
|
||||
use log::info;
|
||||
pub use states::*;
|
||||
|
||||
use crate::structs::{Config, Exit};
|
||||
|
||||
pub fn main(config: &Config, rpc: &Client, exit: &Exit) -> color_eyre::Result<()> {
|
||||
loop {
|
||||
let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
|
||||
|
||||
info!("{block_count} blocks found.");
|
||||
|
||||
let mut databases = Databases::import(config);
|
||||
let mut datasets = Datasets::import(config)?;
|
||||
|
||||
iter_blocks(config, rpc, block_count, exit.clone(), &mut databases, &mut datasets)?;
|
||||
|
||||
if let Some(delay) = config.delay() {
|
||||
sleep(Duration::from_secs(delay))
|
||||
}
|
||||
|
||||
info!("Waiting for a new block...");
|
||||
|
||||
while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
@@ -1,26 +1,29 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::{collections::BTreeMap, fs, path::Path};
|
||||
use std::{collections::BTreeMap, fs};
|
||||
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use itertools::Itertools;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
io::{Json, INPUTS_FOLDER_PATH},
|
||||
structs::{Date, Timestamp, OHLC},
|
||||
utils::{log, retry},
|
||||
io::Json,
|
||||
structs::{Config, Date, Timestamp, OHLC},
|
||||
utils::retry,
|
||||
};
|
||||
|
||||
pub struct Binance;
|
||||
|
||||
impl Binance {
|
||||
pub fn read_har_file() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
log("binance: read har file");
|
||||
pub fn read_har_file(config: &Config) -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
info!("binance: read har file");
|
||||
|
||||
fs::create_dir_all(INPUTS_FOLDER_PATH)?;
|
||||
let path = config.path_inputs();
|
||||
|
||||
let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har");
|
||||
fs::create_dir_all(&path)?;
|
||||
|
||||
let path_binance_har = path.join("binance.har");
|
||||
|
||||
let json: BTreeMap<String, Value> = Json::import(&path_binance_har).unwrap_or_default();
|
||||
|
||||
@@ -104,7 +107,7 @@ impl Binance {
|
||||
}
|
||||
|
||||
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
log("binance: fetch 1mn");
|
||||
info!("binance: fetch 1mn");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -117,33 +120,37 @@ impl Binance {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
// [timestamp, open, high, low, close, volume, ...]
|
||||
let array = value.as_array().unwrap();
|
||||
let array = value.as_array().context("Expect to be array")?;
|
||||
|
||||
let timestamp = (array.first().unwrap().as_u64().unwrap() / 1_000) as u32;
|
||||
let timestamp = (array
|
||||
.first()
|
||||
.context("Expect to have first")?
|
||||
.as_u64()
|
||||
.context("Expect to be convertible to u64")?
|
||||
/ 1_000) as u32;
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect to have index")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect to have &str")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
timestamp,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
@@ -151,7 +158,7 @@ impl Binance {
|
||||
}
|
||||
|
||||
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
log("binance: fetch 1d");
|
||||
info!("binance: fetch 1d");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -164,36 +171,40 @@ impl Binance {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
// [timestamp, open, high, low, close, volume, ...]
|
||||
let array = value.as_array().unwrap();
|
||||
let array = value.as_array().context("Expect to be array")?;
|
||||
|
||||
let date = Timestamp::wrap(
|
||||
(array.first().unwrap().as_u64().unwrap() / 1_000) as u32,
|
||||
let date = Timestamp::from(
|
||||
(array
|
||||
.first()
|
||||
.context("Expect to have first")?
|
||||
.as_u64()
|
||||
.context("Expect to be convertible to u64")?
|
||||
/ 1_000) as u32,
|
||||
)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect to have index")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect to have &str")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
date,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
@@ -2,13 +2,12 @@ use std::{collections::BTreeMap, str::FromStr};
|
||||
|
||||
use chrono::NaiveDate;
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use itertools::Itertools;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
structs::{Date, DateMapChunkId, HeightMapChunkId, OHLC},
|
||||
utils::{log, retry},
|
||||
MapChunkId,
|
||||
structs::{Date, DateMapChunkId, HeightMapChunkId, MapChunkId, OHLC},
|
||||
utils::retry,
|
||||
};
|
||||
|
||||
pub struct Kibo;
|
||||
@@ -28,7 +27,7 @@ impl Kibo {
|
||||
}
|
||||
|
||||
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
|
||||
log("kibo: fetch height prices");
|
||||
info!("kibo: fetch height prices");
|
||||
|
||||
retry(
|
||||
|try_index| {
|
||||
@@ -40,7 +39,7 @@ impl Kibo {
|
||||
))?
|
||||
.json()?;
|
||||
|
||||
Ok(body
|
||||
let vec = body
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("dataset")
|
||||
@@ -53,7 +52,9 @@ impl Kibo {
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(Self::value_to_ohlc)
|
||||
.collect_vec())
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
Ok(vec)
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
@@ -61,7 +62,7 @@ impl Kibo {
|
||||
}
|
||||
|
||||
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
log("kibo: fetch date prices");
|
||||
info!("kibo: fetch date prices");
|
||||
|
||||
retry(
|
||||
|try_index| {
|
||||
@@ -85,28 +86,33 @@ impl Kibo {
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.iter()
|
||||
.map(|(serialized_date, value)| {
|
||||
let date = Date::wrap(NaiveDate::from_str(serialized_date).unwrap());
|
||||
|
||||
(date, Self::value_to_ohlc(value))
|
||||
.map(|(serialized_date, value)| -> color_eyre::Result<_> {
|
||||
let date = Date::wrap(NaiveDate::from_str(serialized_date)?);
|
||||
Ok((date, Self::value_to_ohlc(value)?))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
fn value_to_ohlc(value: &Value) -> OHLC {
|
||||
let ohlc = value.as_object().unwrap();
|
||||
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLC> {
|
||||
let ohlc = value.as_object().context("Expect as_object to work")?;
|
||||
|
||||
let get_value = |key: &str| ohlc.get(key).unwrap().as_f64().unwrap() as f32;
|
||||
let get_value = |key: &str| -> color_eyre::Result<f32> {
|
||||
Ok(ohlc
|
||||
.get(key)
|
||||
.context("Expect get key to work")?
|
||||
.as_f64()
|
||||
.context("Expect as_f64 to work")? as f32)
|
||||
};
|
||||
|
||||
OHLC {
|
||||
open: get_value("open"),
|
||||
high: get_value("high"),
|
||||
low: get_value("low"),
|
||||
close: get_value("close"),
|
||||
}
|
||||
Ok(OHLC {
|
||||
open: get_value("open")?,
|
||||
high: get_value("high")?,
|
||||
low: get_value("low")?,
|
||||
close: get_value("close")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,18 +1,19 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
structs::{Date, Timestamp, OHLC},
|
||||
utils::{log, retry},
|
||||
utils::retry,
|
||||
};
|
||||
|
||||
pub struct Kraken;
|
||||
|
||||
impl Kraken {
|
||||
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
|
||||
log("kraken: fetch 1mn");
|
||||
info!("kraken: fetch 1mn");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -33,32 +34,36 @@ impl Kraken {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
let array = value.as_array().unwrap();
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
let array = value.as_array().context("Expect as_array to work")?;
|
||||
|
||||
let timestamp = array.first().unwrap().as_u64().unwrap() as u32;
|
||||
let timestamp = array
|
||||
.first()
|
||||
.context("Expect first to work")?
|
||||
.as_u64()
|
||||
.expect("Expect as_u64 to work")
|
||||
as u32;
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect get index to work")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect as_str to work")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
timestamp,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
@@ -66,7 +71,7 @@ impl Kraken {
|
||||
}
|
||||
|
||||
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
log("fetch kraken daily");
|
||||
info!("fetch kraken daily");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
@@ -87,33 +92,39 @@ impl Kraken {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
let array = value.as_array().unwrap();
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
let array = value.as_array().context("Expect as_array to work")?;
|
||||
|
||||
let date = Timestamp::wrap(array.first().unwrap().as_u64().unwrap() as u32)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
let date = Timestamp::from(
|
||||
array
|
||||
.first()
|
||||
.context("Expect first to work")?
|
||||
.as_u64()
|
||||
.context("Expect as_u64 to work")?
|
||||
as u32,
|
||||
)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect get index to work")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect as_str to work")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
date,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
@@ -0,0 +1,36 @@
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
fs, io,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use bincode::{Decode, Encode};
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
|
||||
use crate::{io::Serialization, structs::Config};
|
||||
|
||||
pub trait AnyState
|
||||
where
|
||||
Self: Debug + Encode + Decode + Serialize + DeserializeOwned,
|
||||
{
|
||||
fn name<'a>() -> &'a str;
|
||||
|
||||
fn path(config: &Config) -> PathBuf {
|
||||
config.path_states().join(Self::name())
|
||||
}
|
||||
|
||||
fn reset(&mut self, config: &Config) -> color_eyre::Result<(), io::Error> {
|
||||
self.clear();
|
||||
fs::remove_file(Self::path(config))
|
||||
}
|
||||
|
||||
fn import(config: &Config) -> color_eyre::Result<Self> {
|
||||
Serialization::Binary.import(&Self::path(config))
|
||||
}
|
||||
|
||||
fn export(&self, config: &Config) -> color_eyre::Result<()> {
|
||||
Serialization::Binary.export(Path::new(&Self::path(config)), self)
|
||||
}
|
||||
|
||||
fn clear(&mut self);
|
||||
}
|
||||
+1
-1
@@ -3,7 +3,7 @@ use std::ops::AddAssign;
|
||||
use allocative::Allocative;
|
||||
|
||||
use crate::{
|
||||
states::{DurableStates, IsZero, OneShotStates, PriceToValue, UnrealizedState},
|
||||
parser::states::{DurableStates, IsZero, OneShotStates, PriceToValue, UnrealizedState},
|
||||
structs::{Amount, Price},
|
||||
};
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ use derive_deref::{Deref, DerefMut};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
databases::AddressIndexToAddressData,
|
||||
parser::databases::AddressIndexToAddressData,
|
||||
structs::{AddressData, AddressRealizedData, Amount, Price},
|
||||
};
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{
|
||||
states::InputState,
|
||||
parser::states::InputState,
|
||||
structs::{AddressRealizedData, Amount, LiquidityClassification},
|
||||
};
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::states::OneShotStates;
|
||||
use crate::parser::states::OneShotStates;
|
||||
|
||||
use super::SplitByAddressCohort;
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{
|
||||
states::OutputState,
|
||||
parser::states::OutputState,
|
||||
structs::{AddressRealizedData, Amount, LiquidityClassification},
|
||||
};
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{
|
||||
states::RealizedState,
|
||||
parser::states::RealizedState,
|
||||
structs::{AddressRealizedData, LiquidityClassification, Price},
|
||||
};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user