mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-22 12:23:04 -07:00
Compare commits
81 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 098f6de047 | |||
| 1b0f90fd68 | |||
| 12252f407b | |||
| 3b6e3f47ab | |||
| 6a9ac9b025 | |||
| ae6aa4088b | |||
| c08f431180 | |||
| 123c1f56e9 | |||
| 35ac65a864 | |||
| e9f362cc87 | |||
| 65685c23e1 | |||
| 2f74748cea | |||
| f477bd66f3 | |||
| d7d77ae8f0 | |||
| 31110a740d | |||
| b64d8b1d7f | |||
| c46006aacc | |||
| 92f81b1493 | |||
| 70213cfc8f | |||
| 8a82bf5c50 | |||
| 37405384a2 | |||
| 54ea6cc53b | |||
| 339c00d815 | |||
| ea6b4dcde2 | |||
| 2b84623d1e | |||
| c8b3afa56b | |||
| 1348f3c24c | |||
| 62208ce3e1 | |||
| 813b2481de | |||
| 27b924ba61 | |||
| b40170b8ce | |||
| 8bfa9d2734 | |||
| c7cf76d4a8 | |||
| dfd2969b3e | |||
| 0e1866fe1d | |||
| b9ae46b913 | |||
| 06e7284055 | |||
| 93289e8fca | |||
| 130d5057d4 | |||
| be492d5084 | |||
| e0bf1d736f | |||
| 5a6b71cbeb | |||
| e6934cd5e2 | |||
| b5aada0792 | |||
| 165ea83ac3 | |||
| 440a82dee4 | |||
| 9c2d3e5e26 | |||
| 6fb6abcbe5 | |||
| dc449dafd1 | |||
| ecdaeebbfb | |||
| fa958b59bd | |||
| fb3d8521cd | |||
| 608c401cf3 | |||
| 1c3da90a24 | |||
| 34567f3375 | |||
| 51bcbeb48f | |||
| cc0f9c42df | |||
| a11bf5523b | |||
| 1921c3d901 | |||
| d568469e8b | |||
| 20d5c7e8d5 | |||
| 9f289ed9de | |||
| 93ee5e480b | |||
| 98a312701f | |||
| cbcf603b63 | |||
| f976f672cf | |||
| cfc3081e8a | |||
| 99818924ee | |||
| 9bbf3a027f | |||
| 93e01902e3 | |||
| 34919aba05 | |||
| a8ee4cf57f | |||
| b39548b4c6 | |||
| 4217c22ff6 | |||
| 4ab10670c9 | |||
| 2883f88de6 | |||
| e002a61a19 | |||
| 5893376279 | |||
| 411c5e4c4d | |||
| c2a77072d2 | |||
| c8a25934a6 |
@@ -1,2 +0,0 @@
|
||||
[build]
|
||||
rustflags = ["-C", "target-cpu=native"]
|
||||
+2
-20
@@ -3,35 +3,17 @@
|
||||
|
||||
# Builds
|
||||
target
|
||||
dist
|
||||
|
||||
# Copies
|
||||
*\ copy*
|
||||
|
||||
# Ignored
|
||||
/_*
|
||||
_*
|
||||
|
||||
# Editors
|
||||
.vscode
|
||||
.zed
|
||||
|
||||
# Flamegraph
|
||||
flamegraph/
|
||||
flamegraph.svg
|
||||
|
||||
# Benchmarks
|
||||
benches
|
||||
|
||||
# Snapshots
|
||||
snapshots*/
|
||||
|
||||
# Docker
|
||||
docker/kibo
|
||||
|
||||
# Types
|
||||
paths.d.ts
|
||||
|
||||
# Outputs
|
||||
_outputs
|
||||
|
||||
# Logs
|
||||
.log
|
||||
|
||||
+2
-31
@@ -3,38 +3,9 @@
|
||||

|
||||
-->
|
||||
|
||||
# v0.6.0 | WIP | A new beginning
|
||||
# v0.X.0 | WIP | A new beginning
|
||||
|
||||
## Global
|
||||
|
||||
- Completely redesign the back-end
|
||||
|
||||
- Merged parser and server crates into a single project (and thus executable), so now both will run at the same time with a single `cargo run -r` [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
|
||||
- Added `--no-server` and `--no-parser` to disable each if needed
|
||||
- Improved executable parameters
|
||||
- Started using `log` and `env_logger` crates instead of custom code [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
|
||||
- Improved logs
|
||||
- Fixed input being unfocused right after being focused in Brave browser [#9a9ae61](https://github.com/kibo-money/kibo/commit/9a9ae614d07b54c08b7e9c0e2aefe3b52fdb93c5)
|
||||
|
||||
- Reworked server's API code [#6ab0f46]( https://github.com/kibo-money/kibo/commit/6ab0f463119a902a1b7ca9691b54f61543bb8f2f)
|
||||
- New route format: `/api/date-to-realized-price` is now `/api/realized-price?kind=date`
|
||||
- Added status and timing to logs
|
||||
- Updated website packages
|
||||
- Added API support for datasets by timestamp (by merging any dataset by height with the height to timestamp dataset and so it still uses heights as chunk ids) [#ca00f3f](https://github.com/kibo-money/kibo/commit/ca00f3f71526f0c5c16021024fec7e5c6e47221c)
|
||||
- `/api/realized-price?kind=t`
|
||||
- `/api/realized-price?kind=timestamp&chunk=860000`
|
||||
- Created separate crate for indexing called `bindex`
|
||||
- Created a crate a storage engine specialized in storing datasets that have indexes as keys and thus can be represented by an array/vec called `storable-vec`
|
||||
- Removed the need for the `-txindex=1` parameter when starting your Bitcoin Core node as kibo has its own indexes now
|
||||
|
||||
## Git
|
||||
|
||||
Added git tags for each version though Markdown won't display formatted on Github so left the default text
|
||||
|
||||
## Deprecated
|
||||
|
||||
Moved Sanakirja database wrapper to its own crate (`snkrj`) and added a robust auto defragmentation to improve disk usage without the need for user's intervention.
|
||||
Since it's not used anymore it will moved out of the repository relatively soon.
|
||||
Full rewrite
|
||||
|
||||
# [kibo-v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
|
||||
|
||||
Generated
+1458
-331
File diff suppressed because it is too large
Load Diff
+27
-19
@@ -4,7 +4,8 @@ members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.0.39"
|
||||
package.version = "0.0.61"
|
||||
package.homepage = "https://bitcoinresearchkit.org"
|
||||
package.repository = "https://github.com/bitcoinresearchkit/brk"
|
||||
|
||||
[profile.release]
|
||||
@@ -16,34 +17,41 @@ panic = "abort"
|
||||
inherits = "release"
|
||||
|
||||
[workspace.dependencies]
|
||||
arc-swap = "1.7.1"
|
||||
axum = "0.8.4"
|
||||
bincode = { version = "2.0.1", features = ["serde"] }
|
||||
bitcoin = { version = "0.32.6", features = ["serde"] }
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
brk_cli = { version = "0", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0", path = "crates/brk_indexer" }
|
||||
brk_logger = { version = "0", path = "crates/brk_logger" }
|
||||
brk_parser = { version = "0", path = "crates/brk_parser" }
|
||||
brk_query = { version = "0", path = "crates/brk_query" }
|
||||
brk_server = { version = "0", path = "crates/brk_server" }
|
||||
brk_vec = { version = "0", path = "crates/brk_vec" }
|
||||
byteview = "0.7.0"
|
||||
clap = { version = "4.5.38", features = ["string"] }
|
||||
clap_derive = "4.5.32"
|
||||
color-eyre = "0.6.4"
|
||||
brk_bundler = { version = "0.0.61", path = "crates/brk_bundler" }
|
||||
brk_cli = { version = "0.0.61", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0.0.61", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0.0.61", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0.0.61", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0.0.61", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.0.61", path = "crates/brk_indexer" }
|
||||
brk_logger = { version = "0.0.61", path = "crates/brk_logger" }
|
||||
brk_parser = { version = "0.0.61", path = "crates/brk_parser" }
|
||||
brk_query = { version = "0.0.61", path = "crates/brk_query" }
|
||||
brk_server = { version = "0.0.61", path = "crates/brk_server" }
|
||||
brk_state = { version = "0.0.61", path = "crates/brk_state" }
|
||||
brk_store = { version = "0.0.61", path = "crates/brk_store" }
|
||||
brk_vec = { version = "0.0.61", path = "crates/brk_vec" }
|
||||
byteview = "=0.6.1"
|
||||
clap = { version = "4.5.40", features = ["string"] }
|
||||
clap_derive = "4.5.40"
|
||||
color-eyre = "0.6.5"
|
||||
derive_deref = "1.1.1"
|
||||
fjall = "2.10.0"
|
||||
jiff = "0.2.13"
|
||||
fjall = "2.11.0"
|
||||
jiff = "0.2.15"
|
||||
log = { version = "0.4.27" }
|
||||
minreq = { version = "2.13.4", features = ["https", "serde_json"] }
|
||||
rayon = "1.10.0"
|
||||
serde = { version = "1.0.219" }
|
||||
serde_bytes = "0.11.17"
|
||||
serde_derive = "1.0.219"
|
||||
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
|
||||
tabled = "0.19.0"
|
||||
tabled = "0.20.0"
|
||||
tokio = { version = "1.45.1", features = ["rt-multi-thread"] }
|
||||
zerocopy = { version = "0.8.25" }
|
||||
zerocopy-derive = "0.8.25"
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 bitcoinresearchkit, kibo.money
|
||||
Copyright (c) 2025 bitcoinresearchkit, kibo.money, satonomics
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://kibo.money">
|
||||
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
@@ -34,28 +31,22 @@
|
||||
</a>
|
||||
</p>
|
||||
|
||||
> **WARNING**
|
||||
>
|
||||
> This project is still a work in progress and while it's much better in many ways than its previous version ([kibo v0.5](https://github.com/kibo-money/kibo)), it doesn't yet include all of those datasets. If you're interested in having everything right now, please use the latter until feature parity is achieved.
|
||||
>
|
||||
> The explorer part (mempool.space/electrs) is also not viable just yet.
|
||||
>
|
||||
> Stay tuned and please be patient, it's a lot of work !
|
||||
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network.
|
||||
|
||||
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin Core node, enabling users to gain deeper insights into the Bitcoin network.
|
||||
|
||||
In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) and [electrs](https://github.com/romanz/electrs) all in one package with a particular focus on simplicity and the self-hosting experience.
|
||||
In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) (soon) and [electrs](https://github.com/romanz/electrs) (soon) all in one package with a particular focus on simplicity and ease of use.
|
||||
|
||||
The toolkit can be used in various ways to accommodate as many needs as possible:
|
||||
|
||||
- **[Website](https://kibo.money)** \
|
||||
Everyone is welcome to visit [kibo.money](https://kibo.money) which is the official showcase of the suite's capabilities and served by default when running BRK. \
|
||||
Researchers and developers are free to use the API which endpoints documentation can be found [here](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#endpoints). \
|
||||
As a token of gratitude to the community and to stimulate curiosity, both the website and the API are entirely free, allowing anyone to use them.
|
||||
- **[Website](https://bitcoinresearchkit.org)** \
|
||||
Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
|
||||
It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
|
||||
Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
|
||||
- **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#endpoints)** \
|
||||
Researchers and developers are free to use BRK's public API with  dataset variants at your disposal. \
|
||||
Just like the website, it's entirely free, with no authentication or rate-limiting.
|
||||
- **[CLI](https://crates.io/crates/brk_cli)** \
|
||||
Node runners are strongly encouraged to try out and self-host their own instance. \
|
||||
A lot of effort has gone into making this as easy as possible. \
|
||||
For more information visit: [`brk_cli`](https://crates.io/crates/brk_cli)
|
||||
Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \
|
||||
The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end.
|
||||
- **[Crates](https://crates.io/crates/brk)** \
|
||||
Rust developers have access to a wide range crates, each built upon one another with its own specific purpose, enabling independent use and offering great flexibility.
|
||||
PRs are welcome, especially if their goal is to introduce additional datasets.
|
||||
@@ -77,23 +68,18 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
|
||||
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
|
||||
- [`brk_query`](https://crates.io/crates/brk_query): A library that finds requested datasets.
|
||||
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
|
||||
- [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer
|
||||
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
|
||||
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
|
||||
|
||||
Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [kibo.money](https://kibo.money) public instance.
|
||||
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
|
||||
|
||||
## Hosting as a service
|
||||
|
||||
*Soon™*
|
||||
|
||||
If you'd like to have your own instance hosted for you please contact [hosting@bitcoinresearchkit.org](mailto:hosting@bitcoinresearchkit.org).
|
||||
|
||||
- 2 separate dedicated servers (1 GB/s each) with different ISPs and Cloudflare integration for enhanced performance and optimal availability
|
||||
- 99.9% SLA
|
||||
- Configurated for speed (`raw + eager`)
|
||||
- 99.99% SLA
|
||||
- Configured for speed
|
||||
- Updates delivered at your convenience
|
||||
- Direct communication for feature requests and support
|
||||
- Bitcoin Core or Knots with desired version
|
||||
@@ -102,6 +88,12 @@ If you'd like to have your own instance hosted for you please contact [hosting@b
|
||||
|
||||
Pricing: `0.01 BTC / month` *or* `0.1 BTC / year`
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
|
||||
|
||||
Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [kibo.money](https://kibo.money) public instance.
|
||||
|
||||
## Donate
|
||||
|
||||
[`bc1q09 8zsm89 m7kgyz e338vf ejhpdt 92ua9p 3peuve`](bitcoin:bc1q098zsm89m7kgyze338vfejhpdt92ua9p3peuve)
|
||||
|
||||
@@ -3,12 +3,14 @@ name = "brk"
|
||||
description.workspace = true
|
||||
license.workspace = true
|
||||
readme.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
edition.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[features]
|
||||
full = [
|
||||
"bundler",
|
||||
"core",
|
||||
"computer",
|
||||
"exit",
|
||||
@@ -18,8 +20,11 @@ full = [
|
||||
"parser",
|
||||
"query",
|
||||
"server",
|
||||
"state",
|
||||
"store",
|
||||
"vec",
|
||||
]
|
||||
bundler = ["brk_bundler"]
|
||||
core = ["brk_core"]
|
||||
computer = ["brk_computer"]
|
||||
exit = ["brk_exit"]
|
||||
@@ -29,9 +34,12 @@ logger = ["brk_logger"]
|
||||
parser = ["brk_parser"]
|
||||
query = ["brk_query"]
|
||||
server = ["brk_server"]
|
||||
state = ["brk_state"]
|
||||
store = ["brk_store"]
|
||||
vec = ["brk_vec"]
|
||||
|
||||
[dependencies]
|
||||
brk_bundler = { workspace = true, optional = true }
|
||||
brk_cli = { workspace = true }
|
||||
brk_core = { workspace = true, optional = true }
|
||||
brk_computer = { workspace = true, optional = true }
|
||||
@@ -42,6 +50,8 @@ brk_logger = { workspace = true, optional = true }
|
||||
brk_parser = { workspace = true, optional = true }
|
||||
brk_query = { workspace = true, optional = true }
|
||||
brk_server = { workspace = true, optional = true }
|
||||
brk_state = { workspace = true, optional = true }
|
||||
brk_store = { workspace = true, optional = true }
|
||||
brk_vec = { workspace = true, optional = true }
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
fn main() {}
|
||||
@@ -1,5 +1,12 @@
|
||||
#![doc = include_str!(concat!("../", env!("CARGO_PKG_README")))]
|
||||
|
||||
#[cfg(feature = "bundler")]
|
||||
#[doc(inline)]
|
||||
pub use brk_bundler as bundler;
|
||||
|
||||
#[doc(inline)]
|
||||
pub use brk_cli as cli;
|
||||
|
||||
#[cfg(feature = "core")]
|
||||
#[doc(inline)]
|
||||
pub use brk_core as core;
|
||||
@@ -36,6 +43,14 @@ pub use brk_query as query;
|
||||
#[doc(inline)]
|
||||
pub use brk_server as server;
|
||||
|
||||
#[cfg(feature = "state")]
|
||||
#[doc(inline)]
|
||||
pub use brk_state as state;
|
||||
|
||||
#[cfg(feature = "store")]
|
||||
#[doc(inline)]
|
||||
pub use brk_store as store;
|
||||
|
||||
#[cfg(feature = "vec")]
|
||||
#[doc(inline)]
|
||||
pub use brk_vec as vec;
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
[package]
|
||||
name = "brk_bundler"
|
||||
description = "A thin wrapper around rolldown"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
log = { workspace = true }
|
||||
notify = "8.0.0"
|
||||
brk_rolldown = "0.0.1"
|
||||
sugar_path = "1.2.0"
|
||||
tokio = { workspace = true }
|
||||
@@ -0,0 +1,142 @@
|
||||
use std::{fs, io, path::Path, sync::Arc};
|
||||
|
||||
use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
|
||||
use log::error;
|
||||
use notify::{EventKind, RecursiveMode, Watcher};
|
||||
use sugar_path::SugarPath;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<()> {
|
||||
let source_path = websites_path.join(source_folder);
|
||||
let dist_path = websites_path.join("dist");
|
||||
|
||||
let _ = fs::remove_dir_all(&dist_path);
|
||||
copy_dir_all(&source_path, &dist_path)?;
|
||||
|
||||
let source_scripts = format!("./{source_folder}/scripts");
|
||||
let source_entry = format!("{source_scripts}/entry.js");
|
||||
|
||||
let absolute_websites_path = websites_path.absolutize();
|
||||
|
||||
let mut bundler = Bundler::new(BundlerOptions {
|
||||
input: Some(vec![source_entry.into()]),
|
||||
dir: Some("./dist/scripts".to_string()),
|
||||
cwd: Some(absolute_websites_path),
|
||||
minify: Some(RawMinifyOptions::Bool(true)),
|
||||
sourcemap: Some(SourceMapType::File),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
bundler.write().await.unwrap();
|
||||
|
||||
let absolute_source_index_path = source_path.join("index.html").absolutize();
|
||||
let absolute_source_index_path_clone = absolute_source_index_path.clone();
|
||||
let absolute_source_path = source_path.absolutize();
|
||||
let absolute_source_path_clone = absolute_source_path.clone();
|
||||
let absolute_source_scripts_path = websites_path.join(source_scripts).absolutize();
|
||||
let absolute_source_sw_path = source_path.join("service-worker.js").absolutize();
|
||||
let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
|
||||
|
||||
let absolute_dist_entry_path = dist_path.join("scripts/entry.js").absolutize();
|
||||
let absolute_dist_index_path = dist_path.join("index.html").absolutize();
|
||||
let absolute_dist_path = dist_path.absolutize();
|
||||
let absolute_dist_path_clone = absolute_dist_path.clone();
|
||||
let absolute_dist_sw_path = dist_path.join("service-worker.js").absolutize();
|
||||
|
||||
let write_index = move || {
|
||||
let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
|
||||
|
||||
if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js")) {
|
||||
let start = entry.find("main").unwrap();
|
||||
let end = entry.find(".js").unwrap();
|
||||
let main_hashed = &entry[start..end];
|
||||
contents = contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
|
||||
}
|
||||
|
||||
let _ = fs::write(&absolute_dist_index_path, contents);
|
||||
};
|
||||
|
||||
let write_sw = move || {
|
||||
let contents = fs::read_to_string(&absolute_source_sw_path)
|
||||
.unwrap()
|
||||
.replace("__VERSION__", &format!("v{VERSION}"));
|
||||
let _ = fs::write(&absolute_dist_sw_path, contents);
|
||||
};
|
||||
|
||||
write_index();
|
||||
write_sw();
|
||||
|
||||
if !watch {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let write_index_clone = write_index.clone();
|
||||
|
||||
let mut entry_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(_) => write_index_clone(),
|
||||
Err(e) => error!("watch error: {:?}", e),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
entry_watcher
|
||||
.watch(&absolute_dist_entry_path, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let mut source_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(event) => match event.kind {
|
||||
EventKind::Create(_) => event.paths,
|
||||
EventKind::Modify(_) => event.paths,
|
||||
_ => vec![],
|
||||
}
|
||||
.into_iter()
|
||||
.filter(|path| path.starts_with(&absolute_source_path))
|
||||
.filter(|path| !path.starts_with(&absolute_source_scripts_path))
|
||||
.for_each(|source_path| {
|
||||
let suffix = source_path.strip_prefix(&absolute_source_path).unwrap();
|
||||
let dist_path = absolute_dist_path.join(suffix);
|
||||
|
||||
if source_path == absolute_source_index_path_clone {
|
||||
write_index();
|
||||
} else if source_path == absolute_source_sw_path_clone {
|
||||
write_sw();
|
||||
} else {
|
||||
let _ = fs::copy(&source_path, &dist_path);
|
||||
}
|
||||
}),
|
||||
Err(e) => error!("watch error: {:?}", e),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
source_watcher
|
||||
.watch(&absolute_source_path_clone, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let watcher =
|
||||
brk_rolldown::Watcher::new(vec![Arc::new(Mutex::new(bundler))], None).unwrap();
|
||||
|
||||
watcher.start().await;
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir_all(&dst)?;
|
||||
for entry in fs::read_dir(src)? {
|
||||
let entry = entry?;
|
||||
let ty = entry.file_type()?;
|
||||
if ty.is_dir() {
|
||||
copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
|
||||
} else {
|
||||
fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -4,9 +4,11 @@ description = "A command line interface to interact with the full Bitcoin Resear
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
brk_computer = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
@@ -23,7 +25,8 @@ color-eyre = { workspace = true }
|
||||
log = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
tabled = { workspace = true }
|
||||
toml = "0.8.22"
|
||||
tokio = { workspace = true }
|
||||
toml = "0.8.23"
|
||||
|
||||
[[bin]]
|
||||
name = "brk"
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://kibo.money">
|
||||
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::fs;
|
||||
use std::{fs, thread};
|
||||
|
||||
use brk_core::{dot_brk_log_path, dot_brk_path};
|
||||
use brk_query::Params as QueryArgs;
|
||||
@@ -20,9 +20,9 @@ struct Cli {
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Commands {
|
||||
/// Run the indexer, computer and server
|
||||
/// Run the indexer, computer and server, use `run -h` for more information
|
||||
Run(RunConfig),
|
||||
/// Query generated datasets via the `run` command in a similar fashion as the server's API
|
||||
/// Query generated datasets via the `run` command in a similar fashion as the server's API, use `query -h` for more information
|
||||
Query(QueryArgs),
|
||||
}
|
||||
|
||||
@@ -35,8 +35,12 @@ pub fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
Commands::Run(args) => run(args),
|
||||
Commands::Query(args) => query(args),
|
||||
}
|
||||
thread::Builder::new()
|
||||
.stack_size(128 * 1024 * 1024)
|
||||
.spawn(|| match cli.command {
|
||||
Commands::Run(args) => run(args),
|
||||
Commands::Query(args) => query(args),
|
||||
})?
|
||||
.join()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
@@ -8,23 +8,25 @@ use crate::run::RunConfig;
|
||||
pub fn query(params: QueryParams) -> color_eyre::Result<()> {
|
||||
let config = RunConfig::import(None)?;
|
||||
|
||||
let compressed = config.compressed();
|
||||
let format = config.format();
|
||||
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), compressed, config.check_collisions())?;
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed);
|
||||
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
|
||||
computer.import_vecs(&indexer, config.computation())?;
|
||||
|
||||
let query = Query::build(&indexer, &computer);
|
||||
|
||||
let index = Index::try_from(params.index.as_str())?;
|
||||
|
||||
let ids = params.values.iter().map(|s| s.as_str()).collect::<Vec<_>>();
|
||||
let from = params.from();
|
||||
let to = params.to();
|
||||
let format = params.format();
|
||||
|
||||
let res = query.search_and_format(index, &ids, params.from, params.to, params.format)?;
|
||||
let res = query.search_and_format(index, &ids, from, to, format)?;
|
||||
|
||||
if params.format.is_some() {
|
||||
if format.is_some() {
|
||||
println!("{}", res);
|
||||
} else {
|
||||
println!(
|
||||
|
||||
+100
-88
@@ -1,18 +1,18 @@
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
thread::{self, sleep},
|
||||
thread::sleep,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use bitcoincore_rpc::{self, Auth, Client, RpcApi};
|
||||
use brk_computer::Computer;
|
||||
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
|
||||
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::rpc::{self, Auth, Client, RpcApi};
|
||||
use brk_server::{Server, Website, tokio};
|
||||
use brk_vec::Computation;
|
||||
use brk_server::{Server, Website};
|
||||
use brk_vec::{Computation, Format};
|
||||
use clap_derive::{Parser, ValueEnum};
|
||||
use color_eyre::eyre::eyre;
|
||||
use log::info;
|
||||
@@ -27,9 +27,9 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
|
||||
|
||||
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
|
||||
|
||||
let compressed = config.compressed();
|
||||
let format = config.format();
|
||||
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), compressed, config.check_collisions())?;
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
|
||||
indexer.import_stores()?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
@@ -49,130 +49,143 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let f = move || -> color_eyre::Result<()> {
|
||||
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed);
|
||||
computer.import_stores(&indexer)?;
|
||||
computer.import_vecs(&indexer, config.computation())?;
|
||||
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
|
||||
computer.import_stores(&indexer)?;
|
||||
computer.import_vecs(&indexer, config.computation())?;
|
||||
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?
|
||||
.block_on(async {
|
||||
let server = if config.serve() {
|
||||
let served_indexer = indexer.clone();
|
||||
let served_computer = computer.clone();
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?
|
||||
.block_on(async {
|
||||
let server = if config.serve() {
|
||||
let served_indexer = indexer.clone();
|
||||
let served_computer = computer.clone();
|
||||
|
||||
let server = Server::new(served_indexer, served_computer, config.website())?;
|
||||
let server = Server::new(served_indexer, served_computer, config.website())?;
|
||||
|
||||
let opt = Some(tokio::spawn(async move {
|
||||
server.serve().await.unwrap();
|
||||
}));
|
||||
let watch = config.watch();
|
||||
let opt = Some(tokio::spawn(async move {
|
||||
server.serve(watch).await.unwrap();
|
||||
}));
|
||||
|
||||
sleep(Duration::from_secs(1));
|
||||
sleep(Duration::from_secs(1));
|
||||
|
||||
opt
|
||||
} else {
|
||||
None
|
||||
};
|
||||
opt
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if config.process() {
|
||||
loop {
|
||||
wait_for_synced_node()?;
|
||||
if config.process() {
|
||||
loop {
|
||||
wait_for_synced_node()?;
|
||||
|
||||
let block_count = rpc.get_block_count()?;
|
||||
let block_count = rpc.get_block_count()?;
|
||||
|
||||
info!("{} blocks found.", block_count + 1);
|
||||
info!("{} blocks found.", block_count + 1);
|
||||
|
||||
let starting_indexes = indexer.index(&parser, rpc, &exit)?;
|
||||
let starting_indexes = indexer.index(&parser, rpc, &exit)?;
|
||||
|
||||
computer.compute(&mut indexer, starting_indexes, &exit)?;
|
||||
computer.compute(&mut indexer, starting_indexes, &exit)?;
|
||||
|
||||
if let Some(delay) = config.delay() {
|
||||
sleep(Duration::from_secs(delay))
|
||||
}
|
||||
if let Some(delay) = config.delay() {
|
||||
sleep(Duration::from_secs(delay))
|
||||
}
|
||||
|
||||
info!("Waiting for new blocks...");
|
||||
info!("Waiting for new blocks...");
|
||||
|
||||
while block_count == rpc.get_block_count()? {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
while block_count == rpc.get_block_count()? {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(handle) = server {
|
||||
handle.await.unwrap();
|
||||
}
|
||||
if let Some(handle) = server {
|
||||
handle.await.unwrap();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
};
|
||||
|
||||
thread::Builder::new()
|
||||
.stack_size(128 * 1024 * 1024)
|
||||
.spawn(f)?
|
||||
.join()
|
||||
.unwrap()
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
|
||||
pub struct RunConfig {
|
||||
/// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
bitcoindir: Option<String>,
|
||||
|
||||
/// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
blocksdir: Option<String>,
|
||||
|
||||
/// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
brkdir: Option<String>,
|
||||
|
||||
/// Executed by the runner, default: all, saved
|
||||
/// Activated services, default: all, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
mode: Option<Mode>,
|
||||
services: Option<Services>,
|
||||
|
||||
/// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
|
||||
#[arg(short = 'C', long)]
|
||||
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
computation: Option<Computation>,
|
||||
|
||||
/// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: true, saved
|
||||
#[arg(short, long, value_name = "BOOL")]
|
||||
compressed: Option<bool>,
|
||||
/// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
format: Option<Format>,
|
||||
|
||||
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
|
||||
#[arg(short, long, value_name = "BOOL")]
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short = 'F', long, value_name = "BOOL")]
|
||||
fetch: Option<bool>,
|
||||
|
||||
/// Website served by the server (if active), default: kibo.money, saved
|
||||
/// Website served by the server (if active), default: default, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
website: Option<Website>,
|
||||
|
||||
/// Bitcoin RPC ip, default: localhost, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "IP")]
|
||||
rpcconnect: Option<String>,
|
||||
|
||||
/// Bitcoin RPC port, default: 8332, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PORT")]
|
||||
rpcport: Option<u16>,
|
||||
|
||||
/// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
rpccookiefile: Option<String>,
|
||||
|
||||
/// Bitcoin RPC username, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "USERNAME")]
|
||||
rpcuser: Option<String>,
|
||||
|
||||
/// Bitcoin RPC password, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PASSWORD")]
|
||||
rpcpassword: Option<String>,
|
||||
|
||||
/// Delay between runs, default: 0, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "SECONDS")]
|
||||
delay: Option<u64>,
|
||||
|
||||
/// DEV: Activate to watch the selected website's folder for changes, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
watch: Option<bool>,
|
||||
|
||||
/// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
check_collisions: Option<bool>,
|
||||
}
|
||||
@@ -200,16 +213,20 @@ impl RunConfig {
|
||||
config_saved.brkdir = Some(brkdir);
|
||||
}
|
||||
|
||||
if let Some(mode) = config_args.mode.take() {
|
||||
config_saved.mode = Some(mode);
|
||||
if let Some(services) = config_args.services.take() {
|
||||
config_saved.services = Some(services);
|
||||
}
|
||||
|
||||
if let Some(computation) = config_args.computation.take() {
|
||||
config_saved.computation = Some(computation);
|
||||
}
|
||||
|
||||
if let Some(fetch) = config_args.fetch.take() {
|
||||
config_saved.fetch = Some(fetch);
|
||||
}
|
||||
|
||||
if let Some(compressed) = config_args.compressed.take() {
|
||||
config_saved.compressed = Some(compressed);
|
||||
if let Some(format) = config_args.format.take() {
|
||||
config_saved.format = Some(format);
|
||||
}
|
||||
|
||||
if let Some(website) = config_args.website.take() {
|
||||
@@ -244,6 +261,10 @@ impl RunConfig {
|
||||
config_saved.check_collisions = Some(check_collisions);
|
||||
}
|
||||
|
||||
if let Some(watch) = config_args.watch.take() {
|
||||
config_saved.watch = Some(watch);
|
||||
}
|
||||
|
||||
if config_args != RunConfig::default() {
|
||||
dbg!(config_args);
|
||||
panic!("Didn't consume the full config")
|
||||
@@ -256,19 +277,6 @@ impl RunConfig {
|
||||
|
||||
config.write(&path)?;
|
||||
|
||||
// info!("Configuration {{");
|
||||
// info!(" bitcoindir: {:?}", config.bitcoindir);
|
||||
// info!(" brkdir: {:?}", config.brkdir);
|
||||
// info!(" mode: {:?}", config.mode);
|
||||
// info!(" website: {:?}", config.website);
|
||||
// info!(" rpcconnect: {:?}", config.rpcconnect);
|
||||
// info!(" rpcport: {:?}", config.rpcport);
|
||||
// info!(" rpccookiefile: {:?}", config.rpccookiefile);
|
||||
// info!(" rpcuser: {:?}", config.rpcuser);
|
||||
// info!(" rpcpassword: {:?}", config.rpcpassword);
|
||||
// info!(" delay: {:?}", config.delay);
|
||||
// info!("}}");
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
@@ -314,7 +322,7 @@ impl RunConfig {
|
||||
}
|
||||
|
||||
pub fn rpc(&self) -> color_eyre::Result<&'static Client> {
|
||||
Ok(Box::leak(Box::new(rpc::Client::new(
|
||||
Ok(Box::leak(Box::new(Client::new(
|
||||
&format!(
|
||||
"http://{}:{}",
|
||||
self.rpcconnect().unwrap_or(&"localhost".to_string()),
|
||||
@@ -379,13 +387,13 @@ impl RunConfig {
|
||||
}
|
||||
|
||||
pub fn process(&self) -> bool {
|
||||
self.mode
|
||||
.is_none_or(|m| m == Mode::All || m == Mode::Processor)
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Processor)
|
||||
}
|
||||
|
||||
pub fn serve(&self) -> bool {
|
||||
self.mode
|
||||
.is_none_or(|m| m == Mode::All || m == Mode::Server)
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Server)
|
||||
}
|
||||
|
||||
fn path_cookiefile(&self) -> PathBuf {
|
||||
@@ -414,7 +422,7 @@ impl RunConfig {
|
||||
}
|
||||
|
||||
pub fn website(&self) -> Website {
|
||||
self.website.unwrap_or(Website::KiboMoney)
|
||||
self.website.unwrap_or(Website::Default)
|
||||
}
|
||||
|
||||
pub fn fetch(&self) -> bool {
|
||||
@@ -430,13 +438,17 @@ impl RunConfig {
|
||||
self.computation.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn compressed(&self) -> bool {
|
||||
self.compressed.is_none_or(|b| b)
|
||||
pub fn format(&self) -> Format {
|
||||
self.format.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn check_collisions(&self) -> bool {
|
||||
self.check_collisions.is_some_and(|b| b)
|
||||
}
|
||||
|
||||
pub fn watch(&self) -> bool {
|
||||
self.watch.is_some_and(|b| b)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(
|
||||
@@ -453,7 +465,7 @@ impl RunConfig {
|
||||
PartialOrd,
|
||||
Ord,
|
||||
)]
|
||||
pub enum Mode {
|
||||
pub enum Services {
|
||||
#[default]
|
||||
All,
|
||||
Processor,
|
||||
|
||||
@@ -4,22 +4,22 @@ description = "A Bitcoin dataset computer, built on top of brk_indexer"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
bitcoin = { workspace = true }
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
brk_fetcher = { workspace = true }
|
||||
brk_indexer = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_state = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
clap_derive = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
derive_deref = { workspace = true }
|
||||
fjall = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://kibo.money">
|
||||
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
|
||||
@@ -5,8 +5,8 @@ use brk_core::{default_bitcoin_path, default_brk_path};
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::{Parser, rpc};
|
||||
use brk_vec::Computation;
|
||||
use brk_parser::Parser;
|
||||
use brk_vec::{Computation, Format};
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
@@ -15,9 +15,9 @@ pub fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let bitcoin_dir = default_bitcoin_path();
|
||||
|
||||
let rpc = Box::leak(Box::new(rpc::Client::new(
|
||||
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
|
||||
"http://localhost:8332",
|
||||
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
|
||||
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
|
||||
)?));
|
||||
let exit = Exit::new();
|
||||
|
||||
@@ -31,15 +31,15 @@ pub fn main() -> color_eyre::Result<()> {
|
||||
let outputs_dir = _outputs_dir.as_path();
|
||||
// let outputs_dir = Path::new("../../_outputs");
|
||||
|
||||
let compressed = false;
|
||||
let format = Format::Raw;
|
||||
|
||||
let mut indexer = Indexer::new(outputs_dir, compressed, true)?;
|
||||
let mut indexer = Indexer::new(outputs_dir, true)?;
|
||||
indexer.import_stores()?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
let fetcher = Fetcher::import(None)?;
|
||||
|
||||
let mut computer = Computer::new(outputs_dir, Some(fetcher), compressed);
|
||||
let mut computer = Computer::new(outputs_dir, Some(fetcher), format);
|
||||
computer.import_stores(&indexer)?;
|
||||
computer.import_vecs(&indexer, Computation::Lazy)?;
|
||||
|
||||
|
||||
@@ -5,13 +5,12 @@
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use brk_core::Version;
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
pub use brk_parser::rpc;
|
||||
use brk_vec::{AnyCollectableVec, Compressed, Computation};
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format};
|
||||
|
||||
mod states;
|
||||
mod stores;
|
||||
mod utils;
|
||||
mod vecs;
|
||||
@@ -26,17 +25,19 @@ pub struct Computer {
|
||||
fetcher: Option<Fetcher>,
|
||||
vecs: Option<Vecs>,
|
||||
stores: Option<Stores>,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ONE;
|
||||
|
||||
impl Computer {
|
||||
pub fn new(outputs_dir: &Path, fetcher: Option<Fetcher>, compressed: bool) -> Self {
|
||||
pub fn new(outputs_dir: &Path, fetcher: Option<Fetcher>, format: Format) -> Self {
|
||||
Self {
|
||||
path: outputs_dir.to_owned(),
|
||||
fetcher,
|
||||
vecs: None,
|
||||
stores: None,
|
||||
compressed: Compressed::from(compressed),
|
||||
format,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,11 +47,13 @@ impl Computer {
|
||||
computation: Computation,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.vecs = Some(Vecs::import(
|
||||
// TODO: Give self.path, join inside import
|
||||
&self.path.join("vecs/computed"),
|
||||
VERSION + Version::ZERO,
|
||||
indexer,
|
||||
self.fetcher.is_some(),
|
||||
computation,
|
||||
self.compressed,
|
||||
self.format,
|
||||
)?);
|
||||
Ok(())
|
||||
}
|
||||
@@ -59,7 +62,9 @@ impl Computer {
|
||||
/// Clone struct instead
|
||||
pub fn import_stores(&mut self, indexer: &Indexer) -> color_eyre::Result<()> {
|
||||
self.stores = Some(Stores::import(
|
||||
// TODO: Give self.path, join inside import
|
||||
&self.path.join("stores"),
|
||||
VERSION + Version::ZERO,
|
||||
indexer.keyspace(),
|
||||
)?);
|
||||
Ok(())
|
||||
@@ -74,15 +79,10 @@ impl Computer {
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
info!("Computing...");
|
||||
|
||||
self.vecs.as_mut().unwrap().compute(
|
||||
indexer,
|
||||
starting_indexes,
|
||||
self.fetcher.as_mut(),
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
self.vecs
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
#![allow(unused)]
|
||||
|
||||
use brk_core::{Sats, StoredU32};
|
||||
|
||||
pub struct BlockState {
|
||||
utxos: StoredU32,
|
||||
value: Sats,
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
mod block;
|
||||
mod cohort;
|
||||
mod outputs;
|
||||
|
||||
pub use block::*;
|
||||
pub use cohort::*;
|
||||
pub use outputs::*;
|
||||
@@ -1,107 +0,0 @@
|
||||
#![allow(unused)]
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct Outputs<T> {
|
||||
pub all: T,
|
||||
// pub by_term: OutputsByTerm<T>,
|
||||
// pub by_up_to: OutputsByUpTo<T>,
|
||||
// pub by_from: OutputsByFrom<T>,
|
||||
// pub by_range: OutputsByRange<T>,
|
||||
// pub by_epoch: OutputsByEpoch<T>,
|
||||
// pub by_size: OutputsBySize<T>,
|
||||
// pub by_value: OutputsByValue<T>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OutputsByTerm<T> {
|
||||
pub short: T,
|
||||
pub long: T,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OutputsByUpTo<T> {
|
||||
pub _1d: T,
|
||||
pub _1w: T,
|
||||
pub _1m: T,
|
||||
pub _2m: T,
|
||||
pub _3m: T,
|
||||
pub _4m: T,
|
||||
pub _5m: T,
|
||||
pub _6m: T,
|
||||
pub _1y: T,
|
||||
pub _2y: T,
|
||||
pub _3y: T,
|
||||
pub _5y: T,
|
||||
pub _7y: T,
|
||||
pub _10y: T,
|
||||
pub _15y: T,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OutputsByRange<T> {
|
||||
pub _1d_to_1w: T,
|
||||
pub _1w_to_1m: T,
|
||||
pub _1m_to_3m: T,
|
||||
pub _3m_to_6m: T,
|
||||
pub _6m_to_1y: T,
|
||||
pub _1y_to_2y: T,
|
||||
pub _2y_to_3y: T,
|
||||
pub _3y_to_5y: T,
|
||||
pub _5y_to_7y: T,
|
||||
pub _7y_to_10y: T,
|
||||
pub _10y_to_15y: T,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OutputsByFrom<T> {
|
||||
pub _1y: T,
|
||||
pub _2y: T,
|
||||
pub _4y: T,
|
||||
pub _10y: T,
|
||||
pub _15y: T,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OutputsByEpoch<T> {
|
||||
pub _1: T,
|
||||
pub _2: T,
|
||||
pub _3: T,
|
||||
pub _4: T,
|
||||
pub _5: T,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OutputsBySize<T> {
|
||||
pub from_1_to_10: T,
|
||||
pub from_10_to_100: T,
|
||||
pub from_100_to_1_000: T,
|
||||
pub from_1_000_to_10_000: T,
|
||||
pub from_10000_to_100_000: T,
|
||||
pub from_100_000_to_1_000_000: T,
|
||||
pub from_1_000_000_to_10_000_000: T,
|
||||
pub from_10_000_000_to_1btc: T,
|
||||
pub from_1btc_to_10btc: T,
|
||||
pub from_10btc_to_100btc: T,
|
||||
pub from_100btc_to_1_000btc: T,
|
||||
pub from_1_000btc_to_10_000btc: T,
|
||||
pub from_10_000btc_to_100_000btc: T,
|
||||
pub from_100_000btc: T,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OutputsByValue<T> {
|
||||
pub up_to_1cent: T,
|
||||
pub from_1c_to_10c: T,
|
||||
pub from_10c_to_1d: T,
|
||||
pub from_1d_to_10d: T,
|
||||
pub from_10usd_to_100usd: T,
|
||||
pub from_100usd_to_1_000usd: T,
|
||||
pub from_1_000usd_to_10_000usd: T,
|
||||
pub from_10_000usd_to_100_000usd: T,
|
||||
pub from_100_000usd_to_1_000_000usd: T,
|
||||
pub from_1_000_000usd_to_10_000_000usd: T,
|
||||
pub from_10_000_000usd_to_100_000_000usd: T,
|
||||
pub from_100_000_000usd_to_1_000_000_000usd: T,
|
||||
pub from_1_000_000_000usd: T,
|
||||
// ...
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::Version;
|
||||
use fjall::TransactionalKeyspace;
|
||||
|
||||
const _VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Stores {
|
||||
// pub address_to_utxos_received: Store<AddressIndexOutputIndex, Unit>,
|
||||
@@ -9,18 +12,18 @@ pub struct Stores {
|
||||
}
|
||||
|
||||
impl Stores {
|
||||
pub fn import(_: &Path, _: &TransactionalKeyspace) -> color_eyre::Result<Self> {
|
||||
pub fn import(_: &Path, _: Version, _: &TransactionalKeyspace) -> color_eyre::Result<Self> {
|
||||
// let address_to_utxos_received = Store::import(
|
||||
// keyspace.clone(),
|
||||
// path,
|
||||
// "address_to_utxos_received",
|
||||
// Version::ZERO,
|
||||
// version + VERSION + Version::ZERO,
|
||||
// )?;
|
||||
// let address_to_utxos_spent = Store::import(
|
||||
// keyspace.clone(),
|
||||
// path,
|
||||
// "address_to_utxos_spent",
|
||||
// Version::ZERO,
|
||||
// version + VERSION + Version::ZERO,
|
||||
// )?;
|
||||
|
||||
Ok(Self {
|
||||
|
||||
@@ -2,12 +2,11 @@ use std::{fs, path::Path};
|
||||
|
||||
use brk_core::{
|
||||
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
|
||||
Timestamp, Weight,
|
||||
Timestamp, Version, Weight,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::bitcoin;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
@@ -15,6 +14,8 @@ use super::{
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub height_to_interval: EagerVec<Height, Timestamp>,
|
||||
@@ -32,30 +33,33 @@ pub struct Vecs {
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
Ok(Self {
|
||||
height_to_interval: EagerVec::forced_import(
|
||||
&path.join("height_to_interval"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"interval",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"timestamp",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_first(),
|
||||
)?,
|
||||
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_interval",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_percentiles()
|
||||
.add_minmax()
|
||||
@@ -65,48 +69,59 @@ impl Vecs {
|
||||
path,
|
||||
"block_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_sum().add_total(),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_weight",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_sum().add_total(),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_size",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_sum().add_total(),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
height_to_vbytes: EagerVec::forced_import(
|
||||
&path.join("height_to_vbytes"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"vbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_vbytes",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_sum().add_total(),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
difficultyepoch_to_timestamp: EagerVec::forced_import(
|
||||
&path.join("difficultyepoch_to_timestamp"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
halvingepoch_to_timestamp: EagerVec::forced_import(
|
||||
&path.join("halvingepoch_to_timestamp"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
@@ -118,7 +133,7 @@ impl Vecs {
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.timeindexes_to_timestamp.compute(
|
||||
self.timeindexes_to_timestamp.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -242,6 +257,8 @@ impl Vecs {
|
||||
self.indexes_to_block_vbytes.vecs(),
|
||||
self.indexes_to_block_weight.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use brk_core::StoredU8;
|
||||
use brk_core::{StoredU8, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyVec, Compressed, Computation, Version};
|
||||
use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
@@ -11,6 +11,8 @@ use super::{
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub _0: ComputedVecsFromHeight<StoredU8>,
|
||||
@@ -22,8 +24,9 @@ pub struct Vecs {
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
@@ -32,32 +35,32 @@ impl Vecs {
|
||||
path,
|
||||
"0",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
_1: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"1",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
_50: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"50",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
_100: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"100",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
})
|
||||
@@ -144,6 +147,8 @@ impl Vecs {
|
||||
self._50.vecs(),
|
||||
self._100.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,12 @@ use std::{fs, path::Path};
|
||||
|
||||
use brk_core::{
|
||||
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
|
||||
OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, WeekIndex, YearIndex,
|
||||
OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, Version, WeekIndex, YearIndex,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format, StoredIndex};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
@@ -66,13 +66,14 @@ pub struct Vecs {
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
const VERSION_IN_SATS: Version = Version::ONE;
|
||||
const VERSION_IN_SATS: Version = Version::ZERO;
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
@@ -82,247 +83,282 @@ impl Vecs {
|
||||
|
||||
Ok(Self {
|
||||
dateindex_to_ohlc_in_cents: EagerVec::forced_import(
|
||||
&fetched_path.join("dateindex_to_ohlc_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
&fetched_path,
|
||||
"ohlc_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("dateindex_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("dateindex_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_close_in_cents: EagerVec::forced_import(
|
||||
&path.join("dateindex_to_close_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"close_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_high_in_cents: EagerVec::forced_import(
|
||||
&path.join("dateindex_to_high_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"high_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_low_in_cents: EagerVec::forced_import(
|
||||
&path.join("dateindex_to_low_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"low_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_open_in_cents: EagerVec::forced_import(
|
||||
&path.join("dateindex_to_open_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"open_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_ohlc_in_cents: EagerVec::forced_import(
|
||||
&fetched_path.join("height_to_ohlc_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
&fetched_path,
|
||||
"ohlc_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("height_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("height_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_close_in_cents: EagerVec::forced_import(
|
||||
&path.join("height_to_close_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"close_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_high_in_cents: EagerVec::forced_import(
|
||||
&path.join("height_to_high_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"high_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_low_in_cents: EagerVec::forced_import(
|
||||
&path.join("height_to_low_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"low_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_open_in_cents: EagerVec::forced_import(
|
||||
&path.join("height_to_open_in_cents"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"open_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"open",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_first(),
|
||||
)?,
|
||||
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"high",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_max(),
|
||||
)?,
|
||||
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"low",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_min(),
|
||||
)?,
|
||||
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"close",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"open_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_first(),
|
||||
)?,
|
||||
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"high_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_max(),
|
||||
)?,
|
||||
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"low_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_min(),
|
||||
)?,
|
||||
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"close_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"open",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_first(),
|
||||
)?,
|
||||
chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"high",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_max(),
|
||||
)?,
|
||||
chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"low",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_min(),
|
||||
)?,
|
||||
chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"close",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"open_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_first(),
|
||||
)?,
|
||||
chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"high_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_max(),
|
||||
)?,
|
||||
chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"low_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_min(),
|
||||
)?,
|
||||
chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
"close_in_sats",
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
weekindex_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("weekindex_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
weekindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("weekindex_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
difficultyepoch_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("difficultyepoch_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("difficultyepoch_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("monthindex_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("monthindex_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
quarterindex_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("quarterindex_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
quarterindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("quarterindex_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("yearindex_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("yearindex_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
// halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ZERO, compressed)?,
|
||||
// halvingepoch_to_ohlc: StorableVec::forced_import(path,
|
||||
// "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
|
||||
decadeindex_to_ohlc: EagerVec::forced_import(
|
||||
&path.join("decadeindex_to_ohlc"),
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
decadeindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
&path.join("decadeindex_to_ohlc_in_sats"),
|
||||
VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
compressed,
|
||||
path,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
@@ -393,8 +429,18 @@ impl Vecs {
|
||||
self.dateindex_to_ohlc_in_cents.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&indexes.dateindex_to_date,
|
||||
|(di, d, ..)| {
|
||||
let ohlc = fetcher.get_date(d).unwrap();
|
||||
|(di, d, this)| {
|
||||
let mut ohlc = fetcher.get_date(d).unwrap();
|
||||
if let Some(prev) = di.decremented() {
|
||||
let prev_open = *this
|
||||
.get_or_read(prev, &this.mmap().load())
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.close;
|
||||
*ohlc.open = prev_open;
|
||||
*ohlc.high = (*ohlc.high).max(prev_open);
|
||||
*ohlc.low = (*ohlc.low).min(prev_open);
|
||||
}
|
||||
(di, ohlc)
|
||||
},
|
||||
exit,
|
||||
@@ -435,7 +481,7 @@ impl Vecs {
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_close.compute(
|
||||
self.timeindexes_to_close.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -450,7 +496,7 @@ impl Vecs {
|
||||
},
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_high.compute(
|
||||
self.timeindexes_to_high.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -465,7 +511,7 @@ impl Vecs {
|
||||
},
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_low.compute(
|
||||
self.timeindexes_to_low.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -480,7 +526,7 @@ impl Vecs {
|
||||
},
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_open.compute(
|
||||
self.timeindexes_to_open.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -748,7 +794,7 @@ impl Vecs {
|
||||
},
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_open_in_sats.compute(
|
||||
self.timeindexes_to_open_in_sats.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -756,14 +802,14 @@ impl Vecs {
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&self.timeindexes_to_open.dateindex,
|
||||
self.timeindexes_to_open.dateindex.as_ref().unwrap(),
|
||||
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_high_in_sats.compute(
|
||||
self.timeindexes_to_high_in_sats.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -771,14 +817,14 @@ impl Vecs {
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&self.timeindexes_to_low.dateindex,
|
||||
self.timeindexes_to_low.dateindex.as_ref().unwrap(),
|
||||
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_low_in_sats.compute(
|
||||
self.timeindexes_to_low_in_sats.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -786,14 +832,14 @@ impl Vecs {
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&self.timeindexes_to_high.dateindex,
|
||||
self.timeindexes_to_high.dateindex.as_ref().unwrap(),
|
||||
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.timeindexes_to_close_in_sats.compute(
|
||||
self.timeindexes_to_close_in_sats.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -801,7 +847,7 @@ impl Vecs {
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&self.timeindexes_to_close.dateindex,
|
||||
self.timeindexes_to_close.dateindex.as_ref().unwrap(),
|
||||
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
|
||||
exit,
|
||||
)
|
||||
@@ -828,12 +874,30 @@ impl Vecs {
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let mut dateindex_first_iter = self.timeindexes_to_open_in_sats.dateindex.iter();
|
||||
let mut dateindex_max_iter = self.timeindexes_to_high_in_sats.dateindex.iter();
|
||||
let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter();
|
||||
let mut dateindex_first_iter = self
|
||||
.timeindexes_to_open_in_sats
|
||||
.dateindex
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.iter();
|
||||
let mut dateindex_max_iter = self
|
||||
.timeindexes_to_high_in_sats
|
||||
.dateindex
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.iter();
|
||||
let mut dateindex_min_iter = self
|
||||
.timeindexes_to_low_in_sats
|
||||
.dateindex
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.iter();
|
||||
self.dateindex_to_ohlc_in_sats.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&self.timeindexes_to_close_in_sats.dateindex,
|
||||
self.timeindexes_to_close_in_sats
|
||||
.dateindex
|
||||
.as_ref()
|
||||
.unwrap(),
|
||||
|(i, close, ..)| {
|
||||
(
|
||||
i,
|
||||
@@ -1097,6 +1161,8 @@ impl Vecs {
|
||||
self.chainindexes_to_low_in_sats.vecs(),
|
||||
self.chainindexes_to_open_in_sats.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{CheckedSub, StoredUsize};
|
||||
use brk_core::{CheckedSub, Result, StoredUsize, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, StoredIndex, StoredType,
|
||||
Version,
|
||||
};
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType};
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
|
||||
use crate::utils::get_percentile;
|
||||
@@ -18,18 +15,18 @@ where
|
||||
I: StoredIndex,
|
||||
T: ComputedType,
|
||||
{
|
||||
first: Option<Box<EagerVec<I, T>>>,
|
||||
average: Option<Box<EagerVec<I, T>>>,
|
||||
sum: Option<Box<EagerVec<I, T>>>,
|
||||
max: Option<Box<EagerVec<I, T>>>,
|
||||
_90p: Option<Box<EagerVec<I, T>>>,
|
||||
_75p: Option<Box<EagerVec<I, T>>>,
|
||||
median: Option<Box<EagerVec<I, T>>>,
|
||||
_25p: Option<Box<EagerVec<I, T>>>,
|
||||
_10p: Option<Box<EagerVec<I, T>>>,
|
||||
min: Option<Box<EagerVec<I, T>>>,
|
||||
last: Option<Box<EagerVec<I, T>>>,
|
||||
total: Option<Box<EagerVec<I, T>>>,
|
||||
pub first: Option<Box<EagerVec<I, T>>>,
|
||||
pub average: Option<Box<EagerVec<I, T>>>,
|
||||
pub sum: Option<Box<EagerVec<I, T>>>,
|
||||
pub max: Option<Box<EagerVec<I, T>>>,
|
||||
pub _90p: Option<Box<EagerVec<I, T>>>,
|
||||
pub _75p: Option<Box<EagerVec<I, T>>>,
|
||||
pub median: Option<Box<EagerVec<I, T>>>,
|
||||
pub _25p: Option<Box<EagerVec<I, T>>>,
|
||||
pub _10p: Option<Box<EagerVec<I, T>>>,
|
||||
pub min: Option<Box<EagerVec<I, T>>>,
|
||||
pub last: Option<Box<EagerVec<I, T>>>,
|
||||
pub cumulative: Option<Box<EagerVec<I, T>>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
@@ -43,64 +40,55 @@ where
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let key = I::to_string().split("::").last().unwrap().to_lowercase();
|
||||
|
||||
let only_one_active = options.is_only_one_active();
|
||||
|
||||
let default = || path.join(format!("{key}_to_{name}"));
|
||||
|
||||
let prefix = |s: &str| path.join(format!("{key}_to_{s}_{name}"));
|
||||
let prefix = |s: &str| format!("{s}_{name}");
|
||||
|
||||
let maybe_prefix = |s: &str| {
|
||||
if only_one_active {
|
||||
default()
|
||||
name.to_string()
|
||||
} else {
|
||||
prefix(s)
|
||||
}
|
||||
};
|
||||
|
||||
let suffix = |s: &str| path.join(format!("{key}_to_{name}_{s}"));
|
||||
let suffix = |s: &str| format!("{name}_{s}");
|
||||
|
||||
let maybe_suffix = |s: &str| {
|
||||
if only_one_active {
|
||||
default()
|
||||
name.to_string()
|
||||
} else {
|
||||
suffix(s)
|
||||
}
|
||||
};
|
||||
|
||||
let version = VERSION + version;
|
||||
|
||||
let s = Self {
|
||||
first: options.first.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_prefix("first"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
last: options.last.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
&path.join(format!("{key}_to_{name}")),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
)
|
||||
.unwrap(),
|
||||
EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(),
|
||||
)
|
||||
}),
|
||||
min: options.min.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("min"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -108,9 +96,10 @@ where
|
||||
max: options.max.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("max"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -118,9 +107,10 @@ where
|
||||
median: options.median.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("median"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -128,9 +118,10 @@ where
|
||||
average: options.average.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("average"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -138,25 +129,32 @@ where
|
||||
sum: options.sum.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("sum"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
total: options.total.then(|| {
|
||||
cumulative: options.cumulative.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed)
|
||||
.unwrap(),
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&prefix("cumulative"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
_90p: options._90p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("90p"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -164,9 +162,10 @@ where
|
||||
_75p: options._75p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("75p"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -174,9 +173,10 @@ where
|
||||
_25p: options._25p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("25p"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -184,9 +184,10 @@ where
|
||||
_10p: options._10p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("10p"),
|
||||
version + Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -202,20 +203,22 @@ where
|
||||
source: &impl AnyIterableVec<I, T>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
if self.total.is_none() {
|
||||
if self.cumulative.is_none() {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.validate_computed_version_or_reset_file(source.version())?;
|
||||
|
||||
let index = self.starting_index(max_from);
|
||||
|
||||
let total_vec = self.total.as_mut().unwrap();
|
||||
let cumulative_vec = self.cumulative.as_mut().unwrap();
|
||||
|
||||
let mut total = index.decremented().map_or(T::from(0_usize), |index| {
|
||||
total_vec.iter().unwrap_get_inner(index)
|
||||
let mut cumulative = index.decremented().map_or(T::from(0_usize), |index| {
|
||||
cumulative_vec.iter().unwrap_get_inner(index)
|
||||
});
|
||||
source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
|
||||
total = total.clone() + v.into_inner();
|
||||
total_vec.forced_push_at(i, total.clone(), exit)
|
||||
cumulative = cumulative.clone() + v.into_inner();
|
||||
cumulative_vec.forced_push_at(i, cumulative.clone(), exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)?;
|
||||
@@ -243,11 +246,11 @@ where
|
||||
let mut count_indexes_iter = count_indexes.iter();
|
||||
let mut source_iter = source.iter();
|
||||
|
||||
let total_vec = self.total.as_mut();
|
||||
let cumulative_vec = self.cumulative.as_mut();
|
||||
|
||||
let mut total = total_vec.map(|total_vec| {
|
||||
let mut cumulative = cumulative_vec.map(|cumulative_vec| {
|
||||
index.decremented().map_or(T::from(0_usize), |index| {
|
||||
total_vec.iter().unwrap_get_inner(index)
|
||||
cumulative_vec.iter().unwrap_get_inner(index)
|
||||
})
|
||||
});
|
||||
|
||||
@@ -281,8 +284,9 @@ where
|
||||
last.forced_push_at(index, v, exit)?;
|
||||
}
|
||||
|
||||
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
|
||||
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
|
||||
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
|
||||
let needs_average_sum_or_cumulative =
|
||||
needs_sum_or_cumulative || self.average.is_some();
|
||||
let needs_sorted = self.max.is_some()
|
||||
|| self._90p.is_some()
|
||||
|| self._75p.is_some()
|
||||
@@ -290,7 +294,7 @@ where
|
||||
|| self._25p.is_some()
|
||||
|| self._10p.is_some()
|
||||
|| self.min.is_some();
|
||||
let needs_values = needs_sorted || needs_average_sum_or_total;
|
||||
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
|
||||
|
||||
if needs_values {
|
||||
source_iter.set(first_index);
|
||||
@@ -351,7 +355,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
if needs_average_sum_or_total {
|
||||
if needs_average_sum_or_cumulative {
|
||||
let len = values.len();
|
||||
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
|
||||
|
||||
@@ -360,15 +364,15 @@ where
|
||||
average.forced_push_at(i, avg, exit)?;
|
||||
}
|
||||
|
||||
if needs_sum_or_total {
|
||||
if needs_sum_or_cumulative {
|
||||
if let Some(sum_vec) = self.sum.as_mut() {
|
||||
sum_vec.forced_push_at(i, sum.clone(), exit)?;
|
||||
}
|
||||
|
||||
if let Some(total_vec) = self.total.as_mut() {
|
||||
let t = total.as_ref().unwrap().clone() + sum;
|
||||
total.replace(t.clone());
|
||||
total_vec.forced_push_at(i, t, exit)?;
|
||||
if let Some(cumulative_vec) = self.cumulative.as_mut() {
|
||||
let t = cumulative.as_ref().unwrap().clone() + sum;
|
||||
cumulative.replace(t.clone());
|
||||
cumulative_vec.forced_push_at(i, t, exit)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -418,9 +422,9 @@ where
|
||||
let mut source_average_iter = source.average.as_ref().map(|f| f.iter());
|
||||
let mut source_sum_iter = source.sum.as_ref().map(|f| f.iter());
|
||||
|
||||
let mut total = self.total.as_mut().map(|total_vec| {
|
||||
let mut cumulative = self.cumulative.as_mut().map(|cumulative_vec| {
|
||||
index.decremented().map_or(T::from(0_usize), |index| {
|
||||
total_vec.iter().unwrap_get_inner(index)
|
||||
cumulative_vec.iter().unwrap_get_inner(index)
|
||||
})
|
||||
});
|
||||
|
||||
@@ -452,10 +456,11 @@ where
|
||||
last.forced_push_at(index, v, exit)?;
|
||||
}
|
||||
|
||||
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
|
||||
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
|
||||
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
|
||||
let needs_average_sum_or_cumulative =
|
||||
needs_sum_or_cumulative || self.average.is_some();
|
||||
let needs_sorted = self.max.is_some() || self.min.is_some();
|
||||
let needs_values = needs_sorted || needs_average_sum_or_total;
|
||||
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
|
||||
|
||||
if needs_values {
|
||||
if needs_sorted {
|
||||
@@ -482,7 +487,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
if needs_average_sum_or_total {
|
||||
if needs_average_sum_or_cumulative {
|
||||
if let Some(average) = self.average.as_mut() {
|
||||
let source_average_iter = source_average_iter.as_mut().unwrap();
|
||||
source_average_iter.set(first_index);
|
||||
@@ -492,14 +497,14 @@ where
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let len = values.len();
|
||||
let total = values.into_iter().fold(T::from(0), |a, b| a + b);
|
||||
// TODO: Multiply by count then divide by total
|
||||
let cumulative = values.into_iter().fold(T::from(0), |a, b| a + b);
|
||||
// TODO: Multiply by count then divide by cumulative
|
||||
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
|
||||
let avg = total / len;
|
||||
let avg = cumulative / len;
|
||||
average.forced_push_at(i, avg, exit)?;
|
||||
}
|
||||
|
||||
if needs_sum_or_total {
|
||||
if needs_sum_or_cumulative {
|
||||
let source_sum_iter = source_sum_iter.as_mut().unwrap();
|
||||
source_sum_iter.set(first_index);
|
||||
let values = source_sum_iter
|
||||
@@ -513,10 +518,10 @@ where
|
||||
sum_vec.forced_push_at(i, sum.clone(), exit)?;
|
||||
}
|
||||
|
||||
if let Some(total_vec) = self.total.as_mut() {
|
||||
let t = total.as_ref().unwrap().clone() + sum;
|
||||
total.replace(t.clone());
|
||||
total_vec.forced_push_at(i, t, exit)?;
|
||||
if let Some(cumulative_vec) = self.cumulative.as_mut() {
|
||||
let t = cumulative.as_ref().unwrap().clone() + sum;
|
||||
cumulative.replace(t.clone());
|
||||
cumulative_vec.forced_push_at(i, t, exit)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -530,7 +535,7 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn starting_index(&self, max_from: I) -> I {
|
||||
pub fn starting_index(&self, max_from: I) -> I {
|
||||
max_from.min(I::from(
|
||||
self.vecs().into_iter().map(|v| v.len()).min().unwrap(),
|
||||
))
|
||||
@@ -576,8 +581,8 @@ where
|
||||
self.last.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_total(&self) -> &EagerVec<I, T> {
|
||||
self.total.as_ref().unwrap()
|
||||
pub fn unwrap_cumulative(&self) -> &EagerVec<I, T> {
|
||||
self.cumulative.as_ref().unwrap()
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
@@ -604,8 +609,8 @@ where
|
||||
if let Some(sum) = self.sum.as_ref() {
|
||||
v.push(sum.as_ref());
|
||||
}
|
||||
if let Some(total) = self.total.as_ref() {
|
||||
v.push(total.as_ref());
|
||||
if let Some(cumulative) = self.cumulative.as_ref() {
|
||||
v.push(cumulative.as_ref());
|
||||
}
|
||||
if let Some(_90p) = self._90p.as_ref() {
|
||||
v.push(_90p.as_ref());
|
||||
@@ -645,8 +650,8 @@ where
|
||||
if let Some(sum) = self.sum.as_mut() {
|
||||
sum.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(total) = self.total.as_mut() {
|
||||
total.safe_flush(exit)?;
|
||||
if let Some(cumulative) = self.cumulative.as_mut() {
|
||||
cumulative.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(_90p) = self._90p.as_mut() {
|
||||
_90p.safe_flush(exit)?;
|
||||
@@ -664,7 +669,7 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
|
||||
pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
|
||||
if let Some(first) = self.first.as_mut() {
|
||||
first.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
@@ -686,8 +691,8 @@ where
|
||||
if let Some(sum) = self.sum.as_mut() {
|
||||
sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(total) = self.total.as_mut() {
|
||||
total.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
if let Some(cumulative) = self.cumulative.as_mut() {
|
||||
cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(_90p) = self._90p.as_mut() {
|
||||
_90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
@@ -719,7 +724,7 @@ pub struct StorableVecGeneatorOptions {
|
||||
min: bool,
|
||||
first: bool,
|
||||
last: bool,
|
||||
total: bool,
|
||||
cumulative: bool,
|
||||
}
|
||||
|
||||
impl StorableVecGeneatorOptions {
|
||||
@@ -783,8 +788,8 @@ impl StorableVecGeneatorOptions {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_total(mut self) -> Self {
|
||||
self.total = true;
|
||||
pub fn add_cumulative(mut self) -> Self {
|
||||
self.cumulative = true;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -843,8 +848,8 @@ impl StorableVecGeneatorOptions {
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_total(mut self) -> Self {
|
||||
self.total = false;
|
||||
pub fn rm_cumulative(mut self) -> Self {
|
||||
self.cumulative = false;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -885,7 +890,7 @@ impl StorableVecGeneatorOptions {
|
||||
self.min,
|
||||
self.first,
|
||||
self.last,
|
||||
self.total,
|
||||
self.cumulative,
|
||||
]
|
||||
.iter()
|
||||
.filter(|b| **b)
|
||||
@@ -895,7 +900,7 @@ impl StorableVecGeneatorOptions {
|
||||
|
||||
pub fn copy_self_extra(&self) -> Self {
|
||||
Self {
|
||||
total: self.total,
|
||||
cumulative: self.cumulative,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex};
|
||||
use brk_core::{
|
||||
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, Version, WeekIndex, YearIndex,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
|
||||
|
||||
use crate::vecs::{Indexes, indexes};
|
||||
|
||||
@@ -14,7 +16,7 @@ pub struct ComputedVecsFromDateIndex<T>
|
||||
where
|
||||
T: ComputedType + PartialOrd,
|
||||
{
|
||||
pub dateindex: EagerVec<DateIndex, T>,
|
||||
pub dateindex: Option<EagerVec<DateIndex, T>>,
|
||||
pub dateindex_extra: ComputedVecBuilder<DateIndex, T>,
|
||||
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
|
||||
pub monthindex: ComputedVecBuilder<MonthIndex, T>,
|
||||
@@ -32,44 +34,67 @@ where
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let version = VERSION + version;
|
||||
let dateindex = compute_source.then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
});
|
||||
|
||||
let dateindex_extra = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options.copy_self_extra(),
|
||||
)?;
|
||||
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
Ok(Self {
|
||||
dateindex: EagerVec::forced_import(
|
||||
&path.join(format!("dateindex_to_{name}")),
|
||||
version,
|
||||
compressed,
|
||||
)?,
|
||||
dateindex,
|
||||
dateindex_extra,
|
||||
weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute<F>(
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
@@ -87,14 +112,15 @@ where
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
&mut self.dateindex,
|
||||
self.dateindex.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.compute_rest(indexes, starting_indexes, exit)
|
||||
let dateindex: Option<&EagerVec<DateIndex, T>> = None;
|
||||
self.compute_rest(indexes, starting_indexes, exit, dateindex)
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
@@ -102,25 +128,49 @@ where
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.dateindex_extra
|
||||
.extend(starting_indexes.dateindex, &self.dateindex, exit)?;
|
||||
if let Some(dateindex) = dateindex {
|
||||
self.dateindex_extra
|
||||
.extend(starting_indexes.dateindex, dateindex, exit)?;
|
||||
|
||||
self.weekindex.compute(
|
||||
starting_indexes.weekindex,
|
||||
&self.dateindex,
|
||||
&indexes.weekindex_to_first_dateindex,
|
||||
&indexes.weekindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
self.weekindex.compute(
|
||||
starting_indexes.weekindex,
|
||||
dateindex,
|
||||
&indexes.weekindex_to_first_dateindex,
|
||||
&indexes.weekindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.monthindex.compute(
|
||||
starting_indexes.monthindex,
|
||||
&self.dateindex,
|
||||
&indexes.monthindex_to_first_dateindex,
|
||||
&indexes.monthindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
self.monthindex.compute(
|
||||
starting_indexes.monthindex,
|
||||
dateindex,
|
||||
&indexes.monthindex_to_first_dateindex,
|
||||
&indexes.monthindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
} else {
|
||||
let dateindex = self.dateindex.as_ref().unwrap();
|
||||
|
||||
self.dateindex_extra
|
||||
.extend(starting_indexes.dateindex, dateindex, exit)?;
|
||||
|
||||
self.weekindex.compute(
|
||||
starting_indexes.weekindex,
|
||||
dateindex,
|
||||
&indexes.weekindex_to_first_dateindex,
|
||||
&indexes.weekindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.monthindex.compute(
|
||||
starting_indexes.monthindex,
|
||||
dateindex,
|
||||
&indexes.monthindex_to_first_dateindex,
|
||||
&indexes.monthindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
self.quarterindex.from_aligned(
|
||||
starting_indexes.quarterindex,
|
||||
@@ -151,7 +201,9 @@ where
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
vec![&self.dateindex as &dyn AnyCollectableVec],
|
||||
self.dateindex
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
|
||||
self.dateindex_extra.vecs(),
|
||||
self.weekindex.vecs(),
|
||||
self.monthindex.vecs(),
|
||||
@@ -159,6 +211,8 @@ where
|
||||
self.yearindex.vecs(),
|
||||
self.decadeindex.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{
|
||||
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex,
|
||||
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, Version,
|
||||
WeekIndex, YearIndex,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, Version};
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
|
||||
|
||||
use crate::vecs::{Indexes, indexes};
|
||||
|
||||
@@ -16,7 +17,7 @@ pub struct ComputedVecsFromHeight<T>
|
||||
where
|
||||
T: ComputedType + PartialOrd,
|
||||
{
|
||||
pub height: Option<Box<EagerVec<Height, T>>>,
|
||||
pub height: Option<EagerVec<Height, T>>,
|
||||
pub height_extra: ComputedVecBuilder<Height, T>,
|
||||
pub dateindex: ComputedVecBuilder<DateIndex, T>,
|
||||
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
|
||||
@@ -40,32 +41,28 @@ where
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let version = VERSION + version;
|
||||
|
||||
let height = compute_source.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
&path.join(format!("height_to_{name}")),
|
||||
version,
|
||||
compressed,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
});
|
||||
|
||||
let height_extra = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options.copy_self_extra(),
|
||||
)?;
|
||||
|
||||
let dateindex =
|
||||
ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
|
||||
let dateindex = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
@@ -73,20 +70,48 @@ where
|
||||
height,
|
||||
height_extra,
|
||||
dateindex,
|
||||
weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
difficultyepoch: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
@@ -111,9 +136,7 @@ where
|
||||
)?;
|
||||
|
||||
let height: Option<&EagerVec<Height, T>> = None;
|
||||
self.compute_rest(indexes, starting_indexes, exit, height)?;
|
||||
|
||||
Ok(())
|
||||
self.compute_rest(indexes, starting_indexes, exit, height)
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
@@ -143,7 +166,7 @@ where
|
||||
exit,
|
||||
)?;
|
||||
} else {
|
||||
let height = self.height.as_ref().unwrap().as_ref();
|
||||
let height = self.height.as_ref().unwrap();
|
||||
|
||||
self.height_extra
|
||||
.extend(starting_indexes.height, height, exit)?;
|
||||
@@ -212,7 +235,7 @@ where
|
||||
[
|
||||
self.height
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
|
||||
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
|
||||
self.height_extra.vecs(),
|
||||
self.dateindex.vecs(),
|
||||
self.weekindex.vecs(),
|
||||
@@ -223,6 +246,8 @@ where
|
||||
// self.halvingepoch.vecs(),
|
||||
self.decadeindex.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{DifficultyEpoch, Height};
|
||||
use brk_core::{DifficultyEpoch, Height, Result, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
|
||||
use brk_vec::{AnyCollectableVec, EagerVec, Format};
|
||||
|
||||
use crate::vecs::{Indexes, indexes};
|
||||
|
||||
@@ -31,19 +31,17 @@ where
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let version = VERSION + version;
|
||||
|
||||
let height =
|
||||
EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)?;
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?;
|
||||
|
||||
let height_extra = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options.copy_self_extra(),
|
||||
)?;
|
||||
|
||||
@@ -53,9 +51,13 @@ where
|
||||
height,
|
||||
height_extra,
|
||||
difficultyepoch: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -93,6 +95,8 @@ where
|
||||
self.difficultyepoch.vecs(),
|
||||
// self.halvingepoch.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{
|
||||
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, TxIndex, WeekIndex,
|
||||
YearIndex,
|
||||
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
|
||||
Result, Sats, TxIndex, Version, WeekIndex, YearIndex,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version,
|
||||
AnyCollectableVec, AnyVec, CollectableVec, EagerVec, Format, StoredIndex, VecIterator,
|
||||
};
|
||||
|
||||
use crate::vecs::{Indexes, indexes};
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
|
||||
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
|
||||
|
||||
@@ -43,79 +43,113 @@ where
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let version = VERSION + version;
|
||||
|
||||
let txindex = compute_source.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
&path.join(format!("txindex_to_{name}")),
|
||||
version,
|
||||
compressed,
|
||||
)
|
||||
.unwrap(),
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let height = ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
|
||||
let height = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
Ok(Self {
|
||||
txindex,
|
||||
height,
|
||||
dateindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
|
||||
weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
|
||||
dateindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
difficultyepoch: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path, name, version, compressed, options,
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(
|
||||
&mut EagerVec<TxIndex, T>,
|
||||
&Indexer,
|
||||
&indexes::Vecs,
|
||||
&Indexes,
|
||||
&Exit,
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.txindex.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
// #[allow(unused)]
|
||||
// pub fn compute_all<F>(
|
||||
// &mut self,
|
||||
// indexer: &Indexer,
|
||||
// indexes: &indexes::Vecs,
|
||||
// starting_indexes: &Indexes,
|
||||
// exit: &Exit,
|
||||
// mut compute: F,
|
||||
// ) -> color_eyre::Result<()>
|
||||
// where
|
||||
// F: FnMut(
|
||||
// &mut EagerVec<TxIndex, T>,
|
||||
// &Indexer,
|
||||
// &indexes::Vecs,
|
||||
// &Indexes,
|
||||
// &Exit,
|
||||
// ) -> Result<()>,
|
||||
// {
|
||||
// compute(
|
||||
// self.txindex.as_mut().unwrap(),
|
||||
// indexer,
|
||||
// indexes,
|
||||
// starting_indexes,
|
||||
// exit,
|
||||
// )?;
|
||||
|
||||
let txindex: Option<&StoredVec<TxIndex, T>> = None;
|
||||
self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
|
||||
// let txindex: Option<&StoredVec<TxIndex, T>> = None;
|
||||
// self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
// Ok(())
|
||||
// }
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
@@ -124,7 +158,7 @@ where
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, T>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
) -> Result<()> {
|
||||
if let Some(txindex) = txindex {
|
||||
self.height.compute(
|
||||
starting_indexes.height,
|
||||
@@ -145,6 +179,15 @@ where
|
||||
)?;
|
||||
}
|
||||
|
||||
self.compute_after_height(indexes, starting_indexes, exit)
|
||||
}
|
||||
|
||||
fn compute_after_height(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.dateindex.from_aligned(
|
||||
starting_indexes.dateindex,
|
||||
&self.height,
|
||||
@@ -219,6 +262,368 @@ where
|
||||
// self.halvingepoch.vecs(),
|
||||
self.decadeindex.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
impl ComputedVecsFromTxindex<Bitcoin> {
|
||||
pub fn compute_rest_from_sats(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
sats: &ComputedVecsFromTxindex<Sats>,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, Bitcoin>>,
|
||||
) -> Result<()> {
|
||||
let txindex_version = if let Some(txindex) = txindex {
|
||||
txindex.version()
|
||||
} else {
|
||||
self.txindex.as_ref().unwrap().as_ref().version()
|
||||
};
|
||||
|
||||
self.height
|
||||
.validate_computed_version_or_reset_file(txindex_version)?;
|
||||
|
||||
let starting_index = self.height.starting_index(starting_indexes.height);
|
||||
|
||||
(starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len())
|
||||
.map(Height::from)
|
||||
.try_for_each(|height| -> Result<()> {
|
||||
if let Some(first) = self.height.first.as_mut() {
|
||||
first.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_first()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(average) = self.height.average.as_mut() {
|
||||
average.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_average()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(sum) = self.height.sum.as_mut() {
|
||||
sum.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_sum()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(max) = self.height.max.as_mut() {
|
||||
max.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_max()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_90p) = self.height._90p.as_mut() {
|
||||
_90p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_90p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_75p) = self.height._75p.as_mut() {
|
||||
_75p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_75p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(median) = self.height.median.as_mut() {
|
||||
median.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_median()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_25p) = self.height._25p.as_mut() {
|
||||
_25p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_25p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_10p) = self.height._10p.as_mut() {
|
||||
_10p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_10p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(min) = self.height.min.as_mut() {
|
||||
min.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_min()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(last) = self.height.last.as_mut() {
|
||||
last.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_last()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(cumulative) = self.height.cumulative.as_mut() {
|
||||
cumulative.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_cumulative()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.height.safe_flush(exit)?;
|
||||
|
||||
self.compute_after_height(indexes, starting_indexes, exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl ComputedVecsFromTxindex<Dollars> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_from_bitcoin(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
bitcoin: &ComputedVecsFromTxindex<Bitcoin>,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, Dollars>>,
|
||||
fetched: &fetched::Vecs,
|
||||
) -> Result<()> {
|
||||
let txindex_version = if let Some(txindex) = txindex {
|
||||
txindex.version()
|
||||
} else {
|
||||
self.txindex.as_ref().unwrap().as_ref().version()
|
||||
};
|
||||
|
||||
self.height
|
||||
.validate_computed_version_or_reset_file(txindex_version)?;
|
||||
|
||||
let starting_index = self.height.starting_index(starting_indexes.height);
|
||||
|
||||
let mut close_iter = fetched.chainindexes_to_close.height.into_iter();
|
||||
|
||||
(starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len())
|
||||
.map(Height::from)
|
||||
.try_for_each(|height| -> Result<()> {
|
||||
let price = *close_iter.unwrap_get_inner(height);
|
||||
|
||||
if let Some(first) = self.height.first.as_mut() {
|
||||
first.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_first()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(average) = self.height.average.as_mut() {
|
||||
average.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_average()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(sum) = self.height.sum.as_mut() {
|
||||
sum.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_sum()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(max) = self.height.max.as_mut() {
|
||||
max.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_max()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_90p) = self.height._90p.as_mut() {
|
||||
_90p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_90p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_75p) = self.height._75p.as_mut() {
|
||||
_75p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_75p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(median) = self.height.median.as_mut() {
|
||||
median.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_median()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_25p) = self.height._25p.as_mut() {
|
||||
_25p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_25p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_10p) = self.height._10p.as_mut() {
|
||||
_10p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_10p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(min) = self.height.min.as_mut() {
|
||||
min.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_min()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(last) = self.height.last.as_mut() {
|
||||
last.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_last()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(cumulative) = self.height.cumulative.as_mut() {
|
||||
cumulative.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_cumulative()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.height.safe_flush(exit)?;
|
||||
|
||||
self.compute_after_height(indexes, starting_indexes, exit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,10 @@ mod from_height_strict;
|
||||
mod from_txindex;
|
||||
mod ratio_from_dateindex;
|
||||
mod r#type;
|
||||
mod value_from_dateindex;
|
||||
mod value_from_height;
|
||||
mod value_from_txindex;
|
||||
mod value_height;
|
||||
|
||||
pub use builder::*;
|
||||
pub use from_dateindex::*;
|
||||
@@ -15,5 +17,7 @@ pub use from_height_strict::*;
|
||||
pub use from_txindex::*;
|
||||
pub use ratio_from_dateindex::*;
|
||||
use r#type::*;
|
||||
pub use value_from_dateindex::*;
|
||||
pub use value_from_height::*;
|
||||
pub use value_from_txindex::*;
|
||||
pub use value_height::*;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,176 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
|
||||
|
||||
use super::StorableVecGeneatorOptions;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedValueVecsFromDateIndex {
|
||||
pub sats: ComputedVecsFromDateIndex<Sats>,
|
||||
pub bitcoin: ComputedVecsFromDateIndex<Bitcoin>,
|
||||
pub dollars: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedValueVecsFromDateIndex {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
compute_dollars: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
name,
|
||||
compute_source,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
bitcoin: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_btc"),
|
||||
true,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_usd"),
|
||||
true,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(
|
||||
&mut EagerVec<DateIndex, Sats>,
|
||||
&Indexer,
|
||||
&indexes::Vecs,
|
||||
&Indexes,
|
||||
&Exit,
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.sats.dateindex.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
|
||||
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, dateindex)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
dateindex: Option<&impl CollectableVec<DateIndex, Sats>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(dateindex) = dateindex {
|
||||
self.sats
|
||||
.compute_rest(indexes, starting_indexes, exit, Some(dateindex))?;
|
||||
|
||||
self.bitcoin.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_sats(starting_indexes.dateindex, dateindex, exit)
|
||||
},
|
||||
)?;
|
||||
} else {
|
||||
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
|
||||
|
||||
self.sats
|
||||
.compute_rest(indexes, starting_indexes, exit, dateindex)?;
|
||||
|
||||
self.bitcoin.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_sats(
|
||||
starting_indexes.dateindex,
|
||||
self.sats.dateindex.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
let dateindex_to_bitcoin = self.bitcoin.dateindex.as_ref().unwrap();
|
||||
let dateindex_to_close = fetched
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.timeindexes_to_close
|
||||
.dateindex
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
dollars.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_bitcoin(
|
||||
starting_indexes.dateindex,
|
||||
dateindex_to_bitcoin,
|
||||
dateindex_to_close,
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.sats.vecs(),
|
||||
self.bitcoin.vecs(),
|
||||
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, Dollars, Height, Sats};
|
||||
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version,
|
||||
};
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
|
||||
@@ -18,7 +16,7 @@ pub struct ComputedValueVecsFromHeight {
|
||||
pub dollars: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ONE;
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedValueVecsFromHeight {
|
||||
pub fn forced_import(
|
||||
@@ -26,7 +24,7 @@ impl ComputedValueVecsFromHeight {
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
compute_dollars: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -35,16 +33,16 @@ impl ComputedValueVecsFromHeight {
|
||||
path,
|
||||
name,
|
||||
compute_source,
|
||||
VERSION + version,
|
||||
compressed,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
bitcoin: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_btc"),
|
||||
true,
|
||||
VERSION + version,
|
||||
compressed,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
@@ -52,8 +50,8 @@ impl ComputedValueVecsFromHeight {
|
||||
path,
|
||||
&format!("{name}_in_usd"),
|
||||
true,
|
||||
VERSION + version,
|
||||
compressed,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)
|
||||
.unwrap()
|
||||
@@ -129,15 +127,15 @@ impl ComputedValueVecsFromHeight {
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_sats(
|
||||
starting_indexes.height,
|
||||
self.sats.height.as_ref().unwrap().as_ref(),
|
||||
self.sats.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
let txindex = self.bitcoin.height.as_ref().unwrap().as_ref();
|
||||
let price = &fetched.as_ref().unwrap().chainindexes_to_close.height;
|
||||
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
|
||||
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
dollars.compute_all(
|
||||
@@ -146,7 +144,12 @@ impl ComputedValueVecsFromHeight {
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_bitcoin(starting_indexes.height, txindex, price, exit)
|
||||
v.compute_from_bitcoin(
|
||||
starting_indexes.height,
|
||||
height_to_bitcoin,
|
||||
height_to_close,
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
@@ -160,6 +163,8 @@ impl ComputedValueVecsFromHeight {
|
||||
self.bitcoin.vecs(),
|
||||
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex};
|
||||
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Compressed,
|
||||
Computation, ComputedVecFrom3, LazyVecFrom1, StoredIndex, StoredVec, Version,
|
||||
AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Computation,
|
||||
ComputedVecFrom3, Format, LazyVecFrom1, StoredIndex, StoredVec,
|
||||
};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
@@ -33,7 +33,7 @@ pub struct ComputedValueVecsFromTxindex {
|
||||
pub dollars: Option<ComputedVecsFromTxindex<Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ONE;
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedValueVecsFromTxindex {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -44,25 +44,28 @@ impl ComputedValueVecsFromTxindex {
|
||||
source: Option<BoxedAnyIterableVec<TxIndex, Sats>>,
|
||||
version: Version,
|
||||
computation: Computation,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let compute_source = source.is_none();
|
||||
let compute_dollars = fetched.is_some();
|
||||
|
||||
let name_in_btc = format!("{name}_in_btc");
|
||||
let name_in_usd = format!("{name}_in_usd");
|
||||
|
||||
let sats = ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
name,
|
||||
compute_source,
|
||||
VERSION + version,
|
||||
compressed,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
let bitcoin_txindex = LazyVecFrom1::init(
|
||||
"txindex_to_{name}_in_btc",
|
||||
VERSION + version,
|
||||
&name_in_btc,
|
||||
version + VERSION,
|
||||
source.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s),
|
||||
|txindex: TxIndex, iter| {
|
||||
iter.next_at(txindex.unwrap_to_usize()).map(|(_, value)| {
|
||||
@@ -74,10 +77,10 @@ impl ComputedValueVecsFromTxindex {
|
||||
|
||||
let bitcoin = ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_btc"),
|
||||
&name_in_btc,
|
||||
false,
|
||||
VERSION + version,
|
||||
compressed,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
@@ -85,9 +88,9 @@ impl ComputedValueVecsFromTxindex {
|
||||
ComputedVecFrom3::forced_import_or_init_from_3(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_{name}_in_usd",
|
||||
VERSION + version,
|
||||
compressed,
|
||||
&name_in_usd,
|
||||
version + VERSION,
|
||||
format,
|
||||
bitcoin_txindex.boxed_clone(),
|
||||
indexes.txindex_to_height.boxed_clone(),
|
||||
fetched.chainindexes_to_close.height.boxed_clone(),
|
||||
@@ -120,10 +123,10 @@ impl ComputedValueVecsFromTxindex {
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_usd"),
|
||||
&name_in_usd,
|
||||
false,
|
||||
VERSION + version,
|
||||
compressed,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)
|
||||
.unwrap()
|
||||
@@ -177,6 +180,7 @@ impl ComputedValueVecsFromTxindex {
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, Sats>>,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(txindex) = txindex {
|
||||
self.sats
|
||||
@@ -187,25 +191,32 @@ impl ComputedValueVecsFromTxindex {
|
||||
.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
|
||||
}
|
||||
|
||||
self.bitcoin.compute_rest(
|
||||
self.bitcoin.compute_rest_from_sats(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
&self.sats,
|
||||
Some(&self.bitcoin_txindex),
|
||||
)?;
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
let dollars_txindex = self.dollars_txindex.as_mut().unwrap();
|
||||
|
||||
dollars_txindex.compute_if_necessary(starting_indexes.txindex, exit)?;
|
||||
dollars_txindex.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
dollars.compute_rest(
|
||||
dollars.compute_rest_from_bitcoin(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
&self.bitcoin,
|
||||
Some(dollars_txindex),
|
||||
fetched.as_ref().unwrap(),
|
||||
)?;
|
||||
}
|
||||
|
||||
@@ -215,9 +226,15 @@ impl ComputedValueVecsFromTxindex {
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.sats.vecs(),
|
||||
vec![&self.bitcoin_txindex as &dyn AnyCollectableVec],
|
||||
self.bitcoin.vecs(),
|
||||
self.dollars_txindex
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
|
||||
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedHeightValueVecs {
|
||||
pub sats: Option<EagerVec<Height, Sats>>,
|
||||
pub bitcoin: EagerVec<Height, Bitcoin>,
|
||||
pub dollars: Option<EagerVec<Height, Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedHeightValueVecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
compute_dollars: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: compute_source.then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
|
||||
.unwrap()
|
||||
}),
|
||||
bitcoin: EagerVec::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_btc"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_usd"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(
|
||||
&mut EagerVec<Height, Sats>,
|
||||
&Indexer,
|
||||
&indexes::Vecs,
|
||||
&Indexes,
|
||||
&Exit,
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.sats.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let height: Option<&StoredVec<Height, Sats>> = None;
|
||||
self.compute_rest(fetched, starting_indexes, exit, height)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
height: Option<&impl CollectableVec<Height, Sats>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(height) = height {
|
||||
self.bitcoin
|
||||
.compute_from_sats(starting_indexes.height, height, exit)?;
|
||||
} else {
|
||||
self.bitcoin.compute_from_sats(
|
||||
starting_indexes.height,
|
||||
self.sats.as_ref().unwrap(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
let height_to_bitcoin = &self.bitcoin;
|
||||
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
dollars.compute_from_bitcoin(
|
||||
starting_indexes.height,
|
||||
height_to_bitcoin,
|
||||
height_to_close,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
vec![&self.bitcoin as &dyn AnyCollectableVec],
|
||||
self.sats.as_ref().map_or(vec![], |v| vec![v]),
|
||||
self.dollars.as_ref().map_or(vec![], |v| vec![v]),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,9 @@
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64};
|
||||
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Compressed, Computation, VecIterator, Version};
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
@@ -11,6 +11,8 @@ use super::{
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
|
||||
@@ -21,8 +23,9 @@ pub struct Vecs {
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
@@ -31,22 +34,24 @@ impl Vecs {
|
||||
path,
|
||||
"difficulty",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"difficultyepoch",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"halvingepoch",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
})
|
||||
@@ -60,7 +65,7 @@ impl Vecs {
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
|
||||
self.indexes_to_difficultyepoch.compute(
|
||||
self.indexes_to_difficultyepoch.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -84,7 +89,7 @@ impl Vecs {
|
||||
)?;
|
||||
|
||||
let mut height_to_halvingepoch_iter = indexes.height_to_halvingepoch.into_iter();
|
||||
self.indexes_to_halvingepoch.compute(
|
||||
self.indexes_to_halvingepoch.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -123,6 +128,8 @@ impl Vecs {
|
||||
self.indexes_to_difficultyepoch.vecs(),
|
||||
self.indexes_to_halvingepoch.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
use std::{fs, path::Path};
|
||||
use std::{fs, path::Path, thread};
|
||||
|
||||
use brk_core::Version;
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Compressed, Computation};
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format};
|
||||
|
||||
pub mod blocks;
|
||||
pub mod constants;
|
||||
@@ -12,10 +13,13 @@ pub mod grouped;
|
||||
pub mod indexes;
|
||||
pub mod market;
|
||||
pub mod mining;
|
||||
pub mod stateful;
|
||||
pub mod transactions;
|
||||
pub mod utxos;
|
||||
|
||||
pub use indexes::Indexes;
|
||||
use log::info;
|
||||
|
||||
const VERSION: Version = Version::ONE;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
@@ -25,37 +29,87 @@ pub struct Vecs {
|
||||
pub mining: mining::Vecs,
|
||||
pub market: market::Vecs,
|
||||
pub transactions: transactions::Vecs,
|
||||
// pub utxos: utxos::Vecs,
|
||||
pub stateful: stateful::Vecs,
|
||||
pub fetched: Option<fetched::Vecs>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
indexer: &Indexer,
|
||||
fetch: bool,
|
||||
computation: Computation,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
let indexes = indexes::Vecs::forced_import(path, indexer, computation, compressed)?;
|
||||
let (indexes, fetched) = thread::scope(|s| {
|
||||
let indexes_handle = s.spawn(|| {
|
||||
indexes::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexer,
|
||||
computation,
|
||||
format,
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
let fetched =
|
||||
fetch.then(|| fetched::Vecs::forced_import(path, computation, compressed).unwrap());
|
||||
let fetch_handle = s.spawn(|| {
|
||||
fetch.then(|| {
|
||||
fetched::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)
|
||||
.unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
(indexes_handle.join().unwrap(), fetch_handle.join().unwrap())
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
blocks: blocks::Vecs::forced_import(path, computation, compressed)?,
|
||||
mining: mining::Vecs::forced_import(path, computation, compressed)?,
|
||||
constants: constants::Vecs::forced_import(path, computation, compressed)?,
|
||||
market: market::Vecs::forced_import(path, computation, compressed)?,
|
||||
// utxos: utxos::Vecs::forced_import(path, computation, compressed)?,
|
||||
blocks: blocks::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
mining: mining::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
constants: constants::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
market: market::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
stateful: stateful::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
fetched.as_ref(),
|
||||
)?,
|
||||
transactions: transactions::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexer,
|
||||
&indexes,
|
||||
computation,
|
||||
compressed,
|
||||
format,
|
||||
fetched.as_ref(),
|
||||
)?,
|
||||
indexes,
|
||||
@@ -70,18 +124,23 @@ impl Vecs {
|
||||
fetcher: Option<&mut Fetcher>,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
info!("Computing indexes...");
|
||||
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
|
||||
|
||||
info!("Computing constants...");
|
||||
self.constants
|
||||
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
|
||||
|
||||
info!("Computing blocks...");
|
||||
self.blocks
|
||||
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
|
||||
|
||||
info!("Computing mining...");
|
||||
self.mining
|
||||
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
|
||||
|
||||
if let Some(fetched) = self.fetched.as_mut() {
|
||||
info!("Computing fetched...");
|
||||
fetched.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
@@ -91,6 +150,7 @@ impl Vecs {
|
||||
)?;
|
||||
}
|
||||
|
||||
info!("Computing transactions...");
|
||||
self.transactions.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
@@ -100,6 +160,7 @@ impl Vecs {
|
||||
)?;
|
||||
|
||||
if let Some(fetched) = self.fetched.as_ref() {
|
||||
info!("Computing market...");
|
||||
self.market.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
@@ -110,6 +171,17 @@ impl Vecs {
|
||||
)?;
|
||||
}
|
||||
|
||||
info!("Computing stateful...");
|
||||
self.stateful.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&self.transactions,
|
||||
self.fetched.as_ref(),
|
||||
&self.market,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -121,8 +193,11 @@ impl Vecs {
|
||||
self.mining.vecs(),
|
||||
self.market.vecs(),
|
||||
self.transactions.vecs(),
|
||||
self.stateful.vecs(),
|
||||
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,258 @@
|
||||
use std::{collections::BTreeMap, ops::ControlFlow};
|
||||
|
||||
use brk_core::{CheckedSub, Dollars, HalvingEpoch, Height, Result, Timestamp};
|
||||
use brk_exit::Exit;
|
||||
use brk_state::{BlockState, OutputFilter, Outputs, Transacted};
|
||||
use brk_vec::StoredIndex;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::vecs::Indexes;
|
||||
|
||||
use super::cohort;
|
||||
|
||||
pub trait OutputCohorts {
|
||||
fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp);
|
||||
fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]);
|
||||
fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>);
|
||||
fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>;
|
||||
}
|
||||
|
||||
impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> {
|
||||
fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
|
||||
if chain_state.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let prev_timestamp = chain_state.last().unwrap().timestamp;
|
||||
|
||||
self.by_date_range
|
||||
.as_mut_vec()
|
||||
.into_par_iter()
|
||||
.for_each(|(filter, v)| {
|
||||
let state = &mut v.state;
|
||||
|
||||
let _ = chain_state
|
||||
.iter()
|
||||
.try_for_each(|block_state| -> ControlFlow<()> {
|
||||
let prev_days_old = block_state
|
||||
.timestamp
|
||||
.difference_in_days_between(prev_timestamp);
|
||||
let days_old = block_state.timestamp.difference_in_days_between(timestamp);
|
||||
|
||||
if prev_days_old == days_old {
|
||||
return ControlFlow::Continue(());
|
||||
}
|
||||
|
||||
let is = filter.contains(days_old);
|
||||
let was = filter.contains(prev_days_old);
|
||||
|
||||
if is && !was {
|
||||
state.increment(&block_state.supply, block_state.price);
|
||||
} else if was && !is {
|
||||
state.decrement(&block_state.supply, block_state.price);
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]) {
|
||||
let mut time_based_vecs = self
|
||||
.by_date_range
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.chain(self.by_epoch.as_mut_vec())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let last_timestamp = chain_state.last().unwrap().timestamp;
|
||||
let current_price = chain_state.last().unwrap().price;
|
||||
|
||||
// dbg!(&height_to_sent);
|
||||
|
||||
height_to_sent.into_iter().for_each(|(height, sent)| {
|
||||
let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
|
||||
let prev_price = block_state.price;
|
||||
|
||||
let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize();
|
||||
|
||||
let days_old = block_state
|
||||
.timestamp
|
||||
.difference_in_days_between(last_timestamp);
|
||||
|
||||
let days_old_foat = block_state
|
||||
.timestamp
|
||||
.difference_in_days_between_float(last_timestamp);
|
||||
|
||||
let older_than_hour =
|
||||
jiff::Timestamp::from(last_timestamp.checked_sub(block_state.timestamp).unwrap())
|
||||
.as_second()
|
||||
>= 60 * 60;
|
||||
|
||||
time_based_vecs
|
||||
.iter_mut()
|
||||
.filter(|(filter, _)| match filter {
|
||||
OutputFilter::From(from) => *from <= days_old,
|
||||
OutputFilter::To(to) => *to > days_old,
|
||||
OutputFilter::Range(range) => range.contains(&days_old),
|
||||
OutputFilter::Epoch(epoch) => *epoch == HalvingEpoch::from(height),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
.for_each(|(_, vecs)| {
|
||||
vecs.state.send(
|
||||
&sent.spendable_supply,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_foat,
|
||||
older_than_hour,
|
||||
);
|
||||
});
|
||||
|
||||
sent.by_type.spendable.as_typed_vec().into_iter().for_each(
|
||||
|(output_type, supply_state)| {
|
||||
self.by_type.get_mut(output_type).1.state.send(
|
||||
supply_state,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_foat,
|
||||
older_than_hour,
|
||||
)
|
||||
},
|
||||
);
|
||||
|
||||
sent.by_size_group
|
||||
.into_iter()
|
||||
.for_each(|(group, supply_state)| {
|
||||
self.by_size_range.get_mut(group).1.state.send(
|
||||
&supply_state,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_foat,
|
||||
older_than_hour,
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
|
||||
let supply_state = received.spendable_supply;
|
||||
|
||||
[
|
||||
&mut self.by_date_range.start_to_1d.1,
|
||||
&mut self.by_epoch.mut_vec_from_height(height).1,
|
||||
]
|
||||
.into_iter()
|
||||
.for_each(|v| {
|
||||
v.state.receive(&supply_state, price);
|
||||
});
|
||||
|
||||
self.by_type
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.for_each(|(filter, vecs)| {
|
||||
let output_type = match filter {
|
||||
OutputFilter::Type(output_type) => *output_type,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
vecs.state.receive(received.by_type.get(output_type), price)
|
||||
});
|
||||
|
||||
received
|
||||
.by_size_group
|
||||
.into_iter()
|
||||
.for_each(|(group, supply_state)| {
|
||||
self.by_size_range
|
||||
.get_mut(group)
|
||||
.1
|
||||
.state
|
||||
.receive(&supply_state, price);
|
||||
});
|
||||
}
|
||||
|
||||
fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
|
||||
let by_date_range = self.by_date_range.as_vec();
|
||||
let by_size_range = self.by_size_range.as_vec();
|
||||
|
||||
[
|
||||
vec![(&mut self.all.1, self.by_epoch.vecs().to_vec())],
|
||||
self.by_from_date
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.map(|(filter, vecs)| {
|
||||
(
|
||||
vecs,
|
||||
by_date_range
|
||||
.into_iter()
|
||||
.filter(|(other, _)| filter.includes(other))
|
||||
.map(|(_, v)| v)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
self.by_up_to_date
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.map(|(filter, vecs)| {
|
||||
(
|
||||
vecs,
|
||||
by_date_range
|
||||
.into_iter()
|
||||
.filter(|(other, _)| filter.includes(other))
|
||||
.map(|(_, v)| v)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
self.by_term
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.map(|(filter, vecs)| {
|
||||
(
|
||||
vecs,
|
||||
by_date_range
|
||||
.into_iter()
|
||||
.filter(|(other, _)| filter.includes(other))
|
||||
.map(|(_, v)| v)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
self.by_from_size
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.map(|(filter, vecs)| {
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.into_iter()
|
||||
.filter(|(other, _)| filter.includes(other))
|
||||
.map(|(_, v)| v)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
self.by_up_to_size
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.map(|(filter, vecs)| {
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.into_iter()
|
||||
.filter(|(other, _)| filter.includes(other))
|
||||
.map(|(_, v)| v)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
]
|
||||
.into_par_iter()
|
||||
.flatten()
|
||||
.try_for_each(|(vecs, stateful)| {
|
||||
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,16 +1,14 @@
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use brk_core::{
|
||||
CheckedSub, Feerate, Height, InputIndex, OutputIndex, Sats, StoredU32, StoredUsize, TxIndex,
|
||||
TxVersion, Weight,
|
||||
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
|
||||
StoredUsize, TxIndex, TxVersion, Version, Weight,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::bitcoin;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Compressed, Computation,
|
||||
ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, StoredIndex, VecIterator,
|
||||
Version,
|
||||
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
|
||||
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -22,6 +20,8 @@ use super::{
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
|
||||
@@ -65,6 +65,7 @@ pub struct Vecs {
|
||||
pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredUsize>,
|
||||
pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredUsize>,
|
||||
pub indexes_to_subsidy: ComputedValueVecsFromHeight,
|
||||
pub indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight,
|
||||
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredUsize>,
|
||||
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredUsize>,
|
||||
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredUsize>,
|
||||
@@ -82,16 +83,17 @@ pub struct Vecs {
|
||||
ComputedVecFrom2<TxIndex, Weight, TxIndex, StoredU32, TxIndex, StoredU32>,
|
||||
pub txindex_to_fee: ComputedVecFrom2<TxIndex, Sats, TxIndex, Sats, TxIndex, Sats>,
|
||||
pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredUsize>,
|
||||
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
|
||||
pub indexes_to_exact_utxo_count: ComputedVecsFromHeight<StoredUsize>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
computation: Computation,
|
||||
compressed: Compressed,
|
||||
format: Format,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let compute_dollars = fetched.is_some();
|
||||
@@ -101,9 +103,9 @@ impl Vecs {
|
||||
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
"inputindex_to_value",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"value",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
indexer.vecs().inputindex_to_outputindex.boxed_clone(),
|
||||
indexer.vecs().outputindex_to_value.boxed_clone(),
|
||||
|index: InputIndex, inputindex_to_outputindex_iter, outputindex_to_value_iter| {
|
||||
@@ -128,9 +130,9 @@ impl Vecs {
|
||||
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_weight",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"weight",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
indexer.vecs().txindex_to_base_size.boxed_clone(),
|
||||
indexer.vecs().txindex_to_total_size.boxed_clone(),
|
||||
|index: TxIndex, txindex_to_base_size_iter, txindex_to_total_size_iter| {
|
||||
@@ -156,9 +158,9 @@ impl Vecs {
|
||||
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_vsize",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"vsize",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
txindex_to_weight.boxed_clone(),
|
||||
|index: TxIndex, iter| {
|
||||
let index = index.unwrap_to_usize();
|
||||
@@ -173,9 +175,9 @@ impl Vecs {
|
||||
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_is_coinbase",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"is_coinbase",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
indexes.txindex_to_height.boxed_clone(),
|
||||
indexer.vecs().height_to_first_txindex.boxed_clone(),
|
||||
|index: TxIndex, txindex_to_height_iter, height_to_first_txindex_iter| {
|
||||
@@ -197,9 +199,9 @@ impl Vecs {
|
||||
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_input_value",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"input_value",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
indexer.vecs().txindex_to_first_inputindex.boxed_clone(),
|
||||
indexes.txindex_to_input_count.boxed_clone(),
|
||||
inputindex_to_value.boxed_clone(),
|
||||
@@ -235,20 +237,20 @@ impl Vecs {
|
||||
// path,
|
||||
// "input_value",
|
||||
// true,
|
||||
// Version::ZERO,
|
||||
// compressed,
|
||||
// version + VERSION + Version::ZERO,
|
||||
// format,
|
||||
// StorableVecGeneatorOptions::default()
|
||||
// .add_average()
|
||||
// .add_sum()
|
||||
// .add_total(),
|
||||
// .add_cumulative(),
|
||||
// )?;
|
||||
|
||||
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_output_value",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"output_value",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
indexer.vecs().txindex_to_first_outputindex.boxed_clone(),
|
||||
indexes.txindex_to_output_count.boxed_clone(),
|
||||
indexer.vecs().outputindex_to_value.boxed_clone(),
|
||||
@@ -284,20 +286,20 @@ impl Vecs {
|
||||
// path,
|
||||
// "output_value",
|
||||
// true,
|
||||
// Version::ZERO,
|
||||
// compressed,
|
||||
// version + VERSION + Version::ZERO,
|
||||
// format,
|
||||
// StorableVecGeneatorOptions::default()
|
||||
// .add_average()
|
||||
// .add_sum()
|
||||
// .add_total(),
|
||||
// .add_cumulative(),
|
||||
// )?;
|
||||
|
||||
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_fee",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"fee",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
txindex_to_input_value.boxed_clone(),
|
||||
txindex_to_output_value.boxed_clone(),
|
||||
|txindex: TxIndex, input_iter, output_iter| {
|
||||
@@ -318,9 +320,9 @@ impl Vecs {
|
||||
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
"txindex_to_feerate",
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
"feerate",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
txindex_to_fee.boxed_clone(),
|
||||
txindex_to_vsize.boxed_clone(),
|
||||
|txindex: TxIndex, fee_iter, vsize_iter| {
|
||||
@@ -340,77 +342,83 @@ impl Vecs {
|
||||
path,
|
||||
"tx_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
"input_count",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
"output_count",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"tx_v1",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_sum().add_total(),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"tx_v2",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_sum().add_total(),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"tx_v3",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_sum().add_total(),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
|
||||
path,
|
||||
"fee",
|
||||
indexes,
|
||||
Some(txindex_to_fee.boxed_clone()),
|
||||
Version::ZERO,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
compressed,
|
||||
format,
|
||||
fetched,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_total()
|
||||
.add_cumulative()
|
||||
.add_percentiles()
|
||||
.add_minmax()
|
||||
.add_average(),
|
||||
@@ -419,8 +427,8 @@ impl Vecs {
|
||||
path,
|
||||
"feerate",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_percentiles()
|
||||
.add_minmax()
|
||||
@@ -430,8 +438,8 @@ impl Vecs {
|
||||
path,
|
||||
"tx_vsize",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_percentiles()
|
||||
.add_minmax()
|
||||
@@ -441,8 +449,8 @@ impl Vecs {
|
||||
path,
|
||||
"tx_weight",
|
||||
false,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_percentiles()
|
||||
.add_minmax()
|
||||
@@ -452,12 +460,12 @@ impl Vecs {
|
||||
path,
|
||||
"subsidy",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total()
|
||||
.add_cumulative()
|
||||
.add_minmax()
|
||||
.add_average(),
|
||||
compute_dollars,
|
||||
@@ -466,178 +474,189 @@ impl Vecs {
|
||||
path,
|
||||
"coinbase",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_total()
|
||||
.add_cumulative()
|
||||
.add_percentiles()
|
||||
.add_minmax()
|
||||
.add_average(),
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
"unclaimed_rewards",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2a_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2ms_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2pk33_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2pk65_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2pkh_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2sh_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2tr_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2wpkh_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"p2wsh_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"opreturn_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"unknownoutput_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"emptyoutput_count",
|
||||
true,
|
||||
Version::ZERO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_average()
|
||||
.add_minmax()
|
||||
.add_percentiles()
|
||||
.add_sum()
|
||||
.add_total(),
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
|
||||
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"utxo_count_bis",
|
||||
"exact_utxo_count",
|
||||
true,
|
||||
Version::TWO,
|
||||
compressed,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
txindex_to_is_coinbase,
|
||||
@@ -718,20 +737,35 @@ impl Vecs {
|
||||
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?;
|
||||
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?;
|
||||
|
||||
self.txindex_to_is_coinbase
|
||||
.compute_if_necessary(starting_indexes.txindex, exit)?;
|
||||
self.txindex_to_is_coinbase.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.txindex_to_weight
|
||||
.compute_if_necessary(starting_indexes.txindex, exit)?;
|
||||
self.txindex_to_weight.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.txindex_to_vsize
|
||||
.compute_if_necessary(starting_indexes.txindex, exit)?;
|
||||
self.txindex_to_vsize.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.inputindex_to_value
|
||||
.compute_if_necessary(starting_indexes.inputindex, exit)?;
|
||||
self.inputindex_to_value.compute_if_necessary(
|
||||
starting_indexes.inputindex,
|
||||
&indexer.vecs().inputindex_to_outputindex,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.txindex_to_output_value
|
||||
.compute_if_necessary(starting_indexes.txindex, exit)?;
|
||||
self.txindex_to_output_value.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
// self.indexes_to_output_value.compute_all(
|
||||
// indexer,
|
||||
@@ -749,8 +783,11 @@ impl Vecs {
|
||||
// },
|
||||
// )?;
|
||||
|
||||
self.txindex_to_input_value
|
||||
.compute_if_necessary(starting_indexes.txindex, exit)?;
|
||||
self.txindex_to_input_value.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
// self.indexes_to_input_value.compute_all(
|
||||
// indexer,
|
||||
@@ -768,12 +805,25 @@ impl Vecs {
|
||||
// },
|
||||
// )?;
|
||||
|
||||
self.txindex_to_fee.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.txindex_to_feerate.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs().txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.indexes_to_fee.compute_rest(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.txindex_to_fee),
|
||||
fetched,
|
||||
)?;
|
||||
|
||||
self.indexes_to_feerate.compute_rest(
|
||||
@@ -844,15 +894,31 @@ impl Vecs {
|
||||
self.indexes_to_fee.sats.height.unwrap_sum().iter();
|
||||
vec.compute_transform(
|
||||
starting_indexes.height,
|
||||
self.indexes_to_coinbase
|
||||
.sats
|
||||
.height
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.as_ref(),
|
||||
|(height, subsidy, ..)| {
|
||||
self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
|
||||
|(height, coinbase, ..)| {
|
||||
let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
|
||||
(height, subsidy.checked_sub(fees).unwrap())
|
||||
(height, coinbase.checked_sub(fees).unwrap())
|
||||
},
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_unclaimed_rewards.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_transform(
|
||||
starting_indexes.height,
|
||||
self.indexes_to_subsidy.sats.height.as_ref().unwrap(),
|
||||
|(height, subsidy, ..)| {
|
||||
let halving = HalvingEpoch::from(height);
|
||||
let expected =
|
||||
Sats::FIFTY_BTC / 2_usize.pow(halving.unwrap_to_usize() as u32);
|
||||
(height, expected.checked_sub(subsidy).unwrap())
|
||||
},
|
||||
exit,
|
||||
)
|
||||
@@ -1039,7 +1105,7 @@ impl Vecs {
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_utxo_count.compute_all(
|
||||
self.indexes_to_exact_utxo_count.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
@@ -1048,16 +1114,16 @@ impl Vecs {
|
||||
let mut input_count_iter = self
|
||||
.indexes_to_input_count
|
||||
.height
|
||||
.unwrap_total()
|
||||
.unwrap_cumulative()
|
||||
.into_iter();
|
||||
let mut opreturn_count_iter = self
|
||||
.indexes_to_opreturn_count
|
||||
.height_extra
|
||||
.unwrap_total()
|
||||
.unwrap_cumulative()
|
||||
.into_iter();
|
||||
v.compute_transform(
|
||||
starting_indexes.height,
|
||||
self.indexes_to_output_count.height.unwrap_total(),
|
||||
self.indexes_to_output_count.height.unwrap_cumulative(),
|
||||
|(h, output_count, ..)| {
|
||||
let input_count = input_count_iter.unwrap_get_inner(h);
|
||||
let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);
|
||||
@@ -1130,8 +1196,11 @@ impl Vecs {
|
||||
self.indexes_to_tx_vsize.vecs(),
|
||||
self.indexes_to_tx_weight.vecs(),
|
||||
self.indexes_to_unknownoutput_count.vecs(),
|
||||
self.indexes_to_utxo_count.vecs(),
|
||||
self.indexes_to_exact_utxo_count.vecs(),
|
||||
self.indexes_to_unclaimed_rewards.vecs(),
|
||||
]
|
||||
.concat()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use brk_core::{CheckedSub, Dollars, Height, Sats, StoredUsize};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyVec, BaseVecIterator, Compressed, Computation, EagerVec, StoredIndex,
|
||||
VecIterator, Version,
|
||||
};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::states::{CohortStates, Outputs};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
|
||||
indexes, transactions,
|
||||
};
|
||||
|
||||
#[derive(Clone, Deref, DerefMut)]
|
||||
pub struct Vecs(Outputs<Vecs_>);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs_ {
|
||||
pub height_to_realized_cap: EagerVec<Height, Dollars>,
|
||||
pub indexes_to_realized_cap: ComputedVecsFromHeight<Dollars>,
|
||||
pub height_to_supply: EagerVec<Height, Sats>,
|
||||
pub indexes_to_supply: ComputedVecsFromHeight<Sats>,
|
||||
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
|
||||
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
_computation: Computation,
|
||||
compressed: Compressed,
|
||||
) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
Ok(Self(Outputs {
|
||||
all: Vecs_ {
|
||||
height_to_realized_cap: EagerVec::forced_import(
|
||||
&path.join("height_to_realized_cap"),
|
||||
VERSION + Version::ZERO,
|
||||
compressed,
|
||||
)?,
|
||||
indexes_to_realized_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"realized_cap",
|
||||
false,
|
||||
VERSION + Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
height_to_supply: EagerVec::forced_import(
|
||||
&path.join("height_to_supply"),
|
||||
VERSION + Version::ZERO,
|
||||
compressed,
|
||||
)?,
|
||||
indexes_to_supply: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"supply",
|
||||
false,
|
||||
VERSION + Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
height_to_utxo_count: EagerVec::forced_import(
|
||||
&path.join("height_to_utxo_count"),
|
||||
VERSION + Version::new(111),
|
||||
compressed,
|
||||
)?,
|
||||
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"utxo_count",
|
||||
false,
|
||||
VERSION + Version::ZERO,
|
||||
compressed,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
transactions: &transactions::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
let indexer_vecs = indexer.vecs();
|
||||
|
||||
let height_to_first_outputindex = &indexer_vecs.height_to_first_outputindex;
|
||||
let height_to_first_inputindex = &indexer_vecs.height_to_first_inputindex;
|
||||
let height_to_output_count = transactions.indexes_to_output_count.height.unwrap_last();
|
||||
let height_to_input_count = transactions.indexes_to_input_count.height.unwrap_last();
|
||||
let inputindex_to_outputindex = &indexer_vecs.inputindex_to_outputindex;
|
||||
let outputindex_to_value = &indexer_vecs.outputindex_to_value;
|
||||
let txindex_to_height = &indexes.txindex_to_height;
|
||||
let outputindex_to_txindex = &indexes.outputindex_to_txindex;
|
||||
|
||||
let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter();
|
||||
let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter();
|
||||
let mut height_to_output_count_iter = height_to_output_count.into_iter();
|
||||
let mut height_to_input_count_iter = height_to_input_count.into_iter();
|
||||
let mut inputindex_to_outputindex_iter = inputindex_to_outputindex.into_iter();
|
||||
let mut outputindex_to_value_iter = outputindex_to_value.into_iter();
|
||||
let mut txindex_to_height_iter = txindex_to_height.into_iter();
|
||||
let mut outputindex_to_txindex_iter = outputindex_to_txindex.into_iter();
|
||||
|
||||
let base_version = Version::ZERO
|
||||
+ height_to_first_outputindex.version()
|
||||
+ height_to_first_inputindex.version()
|
||||
+ height_to_output_count.version()
|
||||
+ height_to_input_count.version()
|
||||
+ inputindex_to_outputindex.version()
|
||||
+ outputindex_to_value.version()
|
||||
+ txindex_to_height.version()
|
||||
+ outputindex_to_txindex.version();
|
||||
|
||||
let height_to_realized_cap = &mut self.0.all.height_to_realized_cap;
|
||||
let height_to_supply = &mut self.0.all.height_to_supply;
|
||||
let height_to_utxo_count = &mut self.0.all.height_to_utxo_count;
|
||||
|
||||
height_to_realized_cap.validate_computed_version_or_reset_file(
|
||||
base_version + height_to_realized_cap.inner_version(),
|
||||
)?;
|
||||
height_to_supply.validate_computed_version_or_reset_file(
|
||||
base_version + height_to_supply.inner_version(),
|
||||
)?;
|
||||
height_to_utxo_count.validate_computed_version_or_reset_file(
|
||||
base_version + height_to_utxo_count.inner_version(),
|
||||
)?;
|
||||
|
||||
let starting_height = [
|
||||
height_to_realized_cap.len(),
|
||||
height_to_supply.len(),
|
||||
height_to_utxo_count.len(),
|
||||
]
|
||||
.into_iter()
|
||||
.map(Height::from)
|
||||
.min()
|
||||
.unwrap()
|
||||
.min(starting_indexes.height);
|
||||
|
||||
let mut states = CohortStates::default();
|
||||
|
||||
if let Some(prev_height) = starting_height.checked_sub(Height::new(1)) {
|
||||
states.realized_cap = height_to_realized_cap
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height);
|
||||
states.supply = height_to_supply.into_iter().unwrap_get_inner(prev_height);
|
||||
states.utxo_count = height_to_utxo_count
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height);
|
||||
}
|
||||
|
||||
(starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len())
|
||||
.map(Height::from)
|
||||
.try_for_each(|height| -> color_eyre::Result<()> {
|
||||
let first_outputindex = height_to_first_outputindex_iter.unwrap_get_inner(height);
|
||||
let first_inputindex = height_to_first_inputindex_iter.unwrap_get_inner(height);
|
||||
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
|
||||
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
// [].concat()
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
@@ -4,20 +4,23 @@ description = "The Core (Structs and Errors) of the Bitcoin Research Kit"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
bincode = { workspace = true }
|
||||
bitcoin = { workspace = true }
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
byteview = { workspace = true }
|
||||
derive_deref = { workspace = true }
|
||||
fjall = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rapidhash = "1.4.0"
|
||||
rlimit = "0.10.2"
|
||||
serde = { workspace = true }
|
||||
serde_derive = { workspace = true }
|
||||
serde_bytes = "0.11.17"
|
||||
serde_bytes = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
zerocopy = { workspace = true }
|
||||
zerocopy-derive = { workspace = true }
|
||||
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://kibo.money">
|
||||
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
|
||||
@@ -1,20 +1,26 @@
|
||||
use std::{
|
||||
fmt::{self, Debug},
|
||||
io,
|
||||
time::SystemTimeError,
|
||||
io, result, time,
|
||||
};
|
||||
|
||||
use crate::Version;
|
||||
|
||||
pub type Result<T, E = Error> = std::result::Result<T, E>;
|
||||
pub type Result<T, E = Error> = result::Result<T, E>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
IO(io::Error),
|
||||
SerdeJson(serde_json::Error),
|
||||
Jiff(jiff::Error),
|
||||
Fjall(fjall::Error),
|
||||
SystemTimeError(time::SystemTimeError),
|
||||
ZeroCopyError,
|
||||
BincodeEncodeError(bincode::error::EncodeError),
|
||||
BincodeDecodeError(bincode::error::DecodeError),
|
||||
|
||||
WrongEndian,
|
||||
DifferentVersion { found: Version, expected: Version },
|
||||
MmapsVecIsTooSmall,
|
||||
IO(io::Error),
|
||||
ZeroCopyError,
|
||||
IndexTooHigh,
|
||||
EmptyVec,
|
||||
IndexTooLow,
|
||||
@@ -24,13 +30,16 @@ pub enum Error {
|
||||
UnsupportedUnflushedState,
|
||||
RangeFromAfterTo(usize, usize),
|
||||
DifferentCompressionMode,
|
||||
SystemTimeError,
|
||||
ToSerdeJsonValueError(serde_json::Error),
|
||||
WrongLength,
|
||||
WrongAddressType,
|
||||
UnindexableDate,
|
||||
|
||||
String(&'static str),
|
||||
}
|
||||
|
||||
impl From<SystemTimeError> for Error {
|
||||
fn from(_: SystemTimeError) -> Self {
|
||||
Self::SystemTimeError
|
||||
impl From<time::SystemTimeError> for Error {
|
||||
fn from(value: time::SystemTimeError) -> Self {
|
||||
Self::SystemTimeError(value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +49,18 @@ impl From<io::Error> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<jiff::Error> for Error {
|
||||
fn from(value: jiff::Error) -> Self {
|
||||
Self::Jiff(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<fjall::Error> for Error {
|
||||
fn from(value: fjall::Error) -> Self {
|
||||
Self::Fjall(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
|
||||
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
|
||||
Self::ZeroCopyError
|
||||
@@ -54,13 +75,34 @@ impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
|
||||
|
||||
impl From<serde_json::Error> for Error {
|
||||
fn from(error: serde_json::Error) -> Self {
|
||||
Self::ToSerdeJsonValueError(error)
|
||||
Self::SerdeJson(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<bincode::error::DecodeError> for Error {
|
||||
fn from(error: bincode::error::DecodeError) -> Self {
|
||||
Self::BincodeDecodeError(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<bincode::error::EncodeError> for Error {
|
||||
fn from(error: bincode::error::EncodeError) -> Self {
|
||||
Self::BincodeEncodeError(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match self {
|
||||
Error::IO(error) => Debug::fmt(&error, f),
|
||||
Error::SystemTimeError(error) => Debug::fmt(&error, f),
|
||||
Error::SerdeJson(error) => Debug::fmt(&error, f),
|
||||
Error::Jiff(error) => Debug::fmt(&error, f),
|
||||
Error::Fjall(error) => Debug::fmt(&error, f),
|
||||
Error::BincodeDecodeError(error) => Debug::fmt(&error, f),
|
||||
Error::BincodeEncodeError(error) => Debug::fmt(&error, f),
|
||||
Error::ZeroCopyError => write!(f, "ZeroCopy error"),
|
||||
|
||||
Error::WrongEndian => write!(f, "Wrong endian"),
|
||||
Error::DifferentVersion { found, expected } => {
|
||||
write!(
|
||||
@@ -69,7 +111,6 @@ impl fmt::Display for Error {
|
||||
)
|
||||
}
|
||||
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
|
||||
Error::IO(error) => Debug::fmt(&error, f),
|
||||
Error::IndexTooHigh => write!(f, "Index too high"),
|
||||
Error::IndexTooLow => write!(f, "Index too low"),
|
||||
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
|
||||
@@ -81,12 +122,17 @@ impl fmt::Display for Error {
|
||||
"Unsupported unflush state, please flush before using this function"
|
||||
)
|
||||
}
|
||||
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
|
||||
Error::SystemTimeError => write!(f, "SystemTimeError"),
|
||||
Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"),
|
||||
Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
|
||||
Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
|
||||
Error::ToSerdeJsonValueError(error) => Debug::fmt(&error, f),
|
||||
Error::WrongLength => write!(f, "Wrong length"),
|
||||
Error::WrongAddressType => write!(f, "Wrong address type"),
|
||||
Error::UnindexableDate => write!(
|
||||
f,
|
||||
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
|
||||
),
|
||||
|
||||
Error::String(s) => write!(f, "{s}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
use std::{
|
||||
fmt::{self, Debug},
|
||||
io,
|
||||
};
|
||||
|
||||
pub type Result<T, E = Error> = std::result::Result<T, E>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
IO(io::Error),
|
||||
Jiff(jiff::Error),
|
||||
ZeroCopyError,
|
||||
WrongLength,
|
||||
WrongAddressType,
|
||||
UnindexableDate,
|
||||
}
|
||||
|
||||
impl From<io::Error> for Error {
|
||||
fn from(value: io::Error) -> Self {
|
||||
Self::IO(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<jiff::Error> for Error {
|
||||
fn from(value: jiff::Error) -> Self {
|
||||
Self::Jiff(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
|
||||
fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
|
||||
Self::ZeroCopyError
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match self {
|
||||
Error::IO(error) => Debug::fmt(&error, f),
|
||||
Error::Jiff(error) => Debug::fmt(&error, f),
|
||||
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
|
||||
Error::WrongLength => write!(f, "Wrong length"),
|
||||
Error::WrongAddressType => write!(f, "Wrong address type"),
|
||||
Error::UnindexableDate => write!(f, "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
@@ -1,9 +1,9 @@
|
||||
#![doc = include_str!("../README.md")]
|
||||
|
||||
mod error;
|
||||
mod enums;
|
||||
mod structs;
|
||||
mod utils;
|
||||
|
||||
pub use error::*;
|
||||
pub use enums::*;
|
||||
pub use structs::*;
|
||||
pub use utils::*;
|
||||
|
||||
@@ -63,9 +63,8 @@ impl From<AddressIndex> for usize {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<ByteView> for AddressIndex {
|
||||
type Error = Error;
|
||||
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
|
||||
impl From<ByteView> for AddressIndex {
|
||||
fn from(value: ByteView) -> Self {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,9 +15,8 @@ pub struct AddressIndexOutputIndex {
|
||||
outputindex: Outputindex,
|
||||
}
|
||||
|
||||
impl TryFrom<ByteView> for AddressIndexOutputIndex {
|
||||
type Error = Error;
|
||||
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
|
||||
impl From<ByteView> for AddressIndexOutputIndex {
|
||||
fn from(value: ByteView) -> Self {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ use derive_deref::Deref;
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
use super::{AddressBytes, OutputType};
|
||||
|
||||
#[derive(
|
||||
@@ -41,10 +39,9 @@ impl From<[u8; 8]> for AddressBytesHash {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<ByteView> for AddressBytesHash {
|
||||
type Error = Error;
|
||||
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
impl From<ByteView> for AddressBytesHash {
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
use std::ops::{Add, Div, Mul};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
ops::{Add, Div, Mul},
|
||||
};
|
||||
|
||||
use serde::Serialize;
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::CheckedSub;
|
||||
|
||||
use super::{Sats, StoredF64};
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Default,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
PartialOrd,
|
||||
FromBytes,
|
||||
Immutable,
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Serialize,
|
||||
)]
|
||||
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
|
||||
pub struct Bitcoin(f64);
|
||||
|
||||
impl Add for Bitcoin {
|
||||
@@ -34,6 +27,21 @@ impl Mul for Bitcoin {
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<usize> for Bitcoin {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: usize) -> Self::Output {
|
||||
Self::from(Sats::from(self) * rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<Bitcoin> for Bitcoin {
|
||||
type Output = StoredF64;
|
||||
fn div(self, rhs: Bitcoin) -> Self::Output {
|
||||
StoredF64::from(self.0 / rhs.0)
|
||||
// Self::from(Sats::from(self) / Sats::from(rhs))
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<usize> for Bitcoin {
|
||||
type Output = Self;
|
||||
fn div(self, rhs: usize) -> Self::Output {
|
||||
@@ -71,11 +79,40 @@ impl From<usize> for Bitcoin {
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for Bitcoin {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => true,
|
||||
(true, false) => false,
|
||||
(false, true) => false,
|
||||
(false, false) => self.0 == other.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for Bitcoin {}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for Bitcoin {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.0.partial_cmp(&other.0).unwrap()
|
||||
impl PartialOrd for Bitcoin {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for Bitcoin {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => Ordering::Equal,
|
||||
(true, false) => Ordering::Less,
|
||||
(false, true) => Ordering::Greater,
|
||||
(false, false) => self.0.partial_cmp(&other.0).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CheckedSub<usize> for Bitcoin {
|
||||
fn checked_sub(self, rhs: usize) -> Option<Self> {
|
||||
Some(Self(self.0 - rhs as f64))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use derive_deref::Deref;
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::{Error, copy_first_8bytes};
|
||||
use crate::copy_first_8bytes;
|
||||
|
||||
use super::BlockHash;
|
||||
|
||||
@@ -35,10 +35,9 @@ impl From<&BlockHash> for BlockHashPrefix {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<ByteView> for BlockHashPrefix {
|
||||
type Error = Error;
|
||||
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
impl From<ByteView> for BlockHashPrefix {
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ use std::ops::{Add, Div, Mul};
|
||||
use serde::Serialize;
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::CheckedSub;
|
||||
|
||||
use super::Dollars;
|
||||
|
||||
#[derive(
|
||||
@@ -20,11 +22,17 @@ use super::Dollars;
|
||||
KnownLayout,
|
||||
Serialize,
|
||||
)]
|
||||
pub struct Cents(u64);
|
||||
pub struct Cents(i64);
|
||||
|
||||
impl Cents {
|
||||
pub const fn mint(value: i64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Dollars> for Cents {
|
||||
fn from(value: Dollars) -> Self {
|
||||
Self((*value * 100.0).round() as u64)
|
||||
Self((*value * 100.0).round() as i64)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,15 +42,45 @@ impl From<Cents> for f64 {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<i64> for Cents {
|
||||
fn from(value: i64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u64> for Cents {
|
||||
fn from(value: u64) -> Self {
|
||||
Self(value)
|
||||
Self(value as i64)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Cents> for usize {
|
||||
fn from(value: Cents) -> Self {
|
||||
if value.0 < 0 {
|
||||
panic!()
|
||||
}
|
||||
value.0 as usize
|
||||
}
|
||||
}
|
||||
|
||||
impl From<usize> for Cents {
|
||||
fn from(value: usize) -> Self {
|
||||
Self(value as i64)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Cents> for i64 {
|
||||
fn from(value: Cents) -> Self {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Cents> for u64 {
|
||||
fn from(value: Cents) -> Self {
|
||||
value.0
|
||||
if value.0 < 0 {
|
||||
panic!("Shouldn't convert neg cents to u64")
|
||||
}
|
||||
value.0 as u64
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,24 +91,34 @@ impl Add for Cents {
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<Cents> for Cents {
|
||||
type Output = Self;
|
||||
fn div(self, rhs: Self) -> Self::Output {
|
||||
Self(self.0 / rhs.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<usize> for Cents {
|
||||
type Output = Self;
|
||||
fn div(self, rhs: usize) -> Self::Output {
|
||||
Self(self.0 / rhs as u64)
|
||||
Self(self.0 / rhs as i64)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u128> for Cents {
|
||||
fn from(value: u128) -> Self {
|
||||
if value > u64::MAX as u128 {
|
||||
panic!("u128 bigger than u64")
|
||||
if value > i64::MAX as u128 {
|
||||
panic!("u128 bigger than i64")
|
||||
}
|
||||
Self(value as u64)
|
||||
Self(value as i64)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Cents> for u128 {
|
||||
fn from(value: Cents) -> Self {
|
||||
if value.0 < 0 {
|
||||
panic!("Shouldn't convert neg cents to u128")
|
||||
}
|
||||
value.0 as u128
|
||||
}
|
||||
}
|
||||
@@ -78,13 +126,26 @@ impl From<Cents> for u128 {
|
||||
impl Mul<Cents> for Cents {
|
||||
type Output = Cents;
|
||||
fn mul(self, rhs: Cents) -> Self::Output {
|
||||
Self(self.0 * rhs.0)
|
||||
Self(self.0.checked_mul(rhs.0).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<i64> for Cents {
|
||||
type Output = Cents;
|
||||
fn mul(self, rhs: i64) -> Self::Output {
|
||||
Self(self.0 * rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<usize> for Cents {
|
||||
type Output = Cents;
|
||||
fn mul(self, rhs: usize) -> Self::Output {
|
||||
Self(self.0 * rhs as u64)
|
||||
Self(self.0 * rhs as i64)
|
||||
}
|
||||
}
|
||||
|
||||
impl CheckedSub for Cents {
|
||||
fn checked_sub(self, rhs: Self) -> Option<Self> {
|
||||
self.0.checked_sub(rhs.0).map(Cents::from)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use std::ops::Add;
|
||||
use std::{
|
||||
fmt,
|
||||
ops::{Add, Rem},
|
||||
};
|
||||
|
||||
use serde::Serialize;
|
||||
// use color_eyre::eyre::eyre;
|
||||
@@ -77,3 +80,16 @@ impl CheckedSub for DateIndex {
|
||||
self.0.checked_sub(rhs.0).map(Self)
|
||||
}
|
||||
}
|
||||
|
||||
impl Rem<usize> for DateIndex {
|
||||
type Output = Self;
|
||||
fn rem(self, rhs: usize) -> Self::Output {
|
||||
Self(self.0 % rhs as u16)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for DateIndex {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,32 +1,39 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
f64,
|
||||
ops::{Add, Div, Mul},
|
||||
ops::{Add, AddAssign, Div, Mul},
|
||||
};
|
||||
|
||||
use bincode::{Decode, Encode};
|
||||
use byteview::ByteView;
|
||||
use derive_deref::Deref;
|
||||
use serde::Serialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use super::{Bitcoin, Cents, Close, Sats, StoredF32, StoredF64};
|
||||
use crate::{CheckedSub, copy_first_8bytes};
|
||||
|
||||
use super::{Bitcoin, Cents, Close, High, Sats, StoredF32, StoredF64};
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Default,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
PartialOrd,
|
||||
Deref,
|
||||
FromBytes,
|
||||
Immutable,
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
Encode,
|
||||
Decode,
|
||||
)]
|
||||
pub struct Dollars(f64);
|
||||
|
||||
impl Dollars {
|
||||
pub const ZERO: Self = Self(0.0);
|
||||
pub const NAN: Self = Self(f64::NAN);
|
||||
|
||||
pub const fn mint(dollars: f64) -> Self {
|
||||
Self(dollars)
|
||||
@@ -69,6 +76,12 @@ impl From<Close<Dollars>> for Dollars {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<High<Dollars>> for Dollars {
|
||||
fn from(value: High<Dollars>) -> Self {
|
||||
Self(value.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<usize> for Dollars {
|
||||
fn from(value: usize) -> Self {
|
||||
Self(value as f64)
|
||||
@@ -85,71 +98,139 @@ impl Add for Dollars {
|
||||
impl Div<Dollars> for Dollars {
|
||||
type Output = StoredF64;
|
||||
fn div(self, rhs: Dollars) -> Self::Output {
|
||||
StoredF64::from(self.0 / rhs.0)
|
||||
if self.is_nan() || rhs == Dollars::ZERO {
|
||||
StoredF64::NAN
|
||||
} else {
|
||||
StoredF64::from(f64::from(self) / f64::from(rhs))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<Close<Dollars>> for Dollars {
|
||||
type Output = StoredF64;
|
||||
fn div(self, rhs: Close<Dollars>) -> Self::Output {
|
||||
StoredF64::from(self.0 / rhs.0)
|
||||
if self.is_nan() || *rhs == Dollars::ZERO {
|
||||
StoredF64::NAN
|
||||
} else {
|
||||
StoredF64::from(f64::from(self) / f64::from(*rhs))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<Dollars> for Close<Dollars> {
|
||||
type Output = StoredF64;
|
||||
fn div(self, rhs: Dollars) -> Self::Output {
|
||||
StoredF64::from(self.0 / rhs.0)
|
||||
if self.is_nan() || rhs == Dollars::ZERO {
|
||||
StoredF64::NAN
|
||||
} else {
|
||||
StoredF64::from(f64::from(*self) / f64::from(rhs))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<usize> for Dollars {
|
||||
type Output = Self;
|
||||
fn div(self, rhs: usize) -> Self::Output {
|
||||
Self::from(Cents::from(self) / rhs)
|
||||
if self.is_nan() || rhs == 0 {
|
||||
Dollars::NAN
|
||||
} else {
|
||||
Self::from(Cents::from(self) / rhs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<Bitcoin> for Dollars {
|
||||
type Output = Self;
|
||||
fn div(self, rhs: Bitcoin) -> Self::Output {
|
||||
Self(f64::from(self) / f64::from(rhs))
|
||||
if self.is_nan() {
|
||||
self
|
||||
} else {
|
||||
Self(f64::from(self) / f64::from(rhs))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for Dollars {}
|
||||
impl Mul<Dollars> for Dollars {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: Dollars) -> Self::Output {
|
||||
Self::from(Cents::from(self) * Cents::from(rhs))
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for Dollars {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.0.partial_cmp(&other.0).unwrap()
|
||||
impl Mul<Close<Dollars>> for Dollars {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: Close<Dollars>) -> Self::Output {
|
||||
Self::from(Cents::from(self) * Cents::from(*rhs))
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<Dollars> for Close<Dollars> {
|
||||
type Output = Dollars;
|
||||
fn mul(self, rhs: Dollars) -> Self::Output {
|
||||
Dollars::from(Cents::from(*self) * Cents::from(rhs))
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<usize> for Close<Dollars> {
|
||||
type Output = Dollars;
|
||||
fn mul(self, rhs: usize) -> Self::Output {
|
||||
Dollars::from(Cents::from(*self) * rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<Bitcoin> for Dollars {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: Bitcoin) -> Self::Output {
|
||||
Self::from(Cents::from(
|
||||
u128::from(Sats::from(rhs)) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC),
|
||||
))
|
||||
self * Sats::from(rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<Bitcoin> for Close<Dollars> {
|
||||
type Output = Dollars;
|
||||
fn mul(self, rhs: Bitcoin) -> Self::Output {
|
||||
*self * Sats::from(rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<Sats> for Dollars {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: Sats) -> Self::Output {
|
||||
if self.is_nan() {
|
||||
self
|
||||
} else {
|
||||
Self::from(Cents::from(
|
||||
u128::from(rhs) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<StoredF32> for Dollars {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: StoredF32) -> Self::Output {
|
||||
if rhs.is_nan() {
|
||||
Self(f64::NAN)
|
||||
if rhs.fract() != 0.0 {
|
||||
Self::from(self.0 * *rhs as f64)
|
||||
} else {
|
||||
Self::from(Cents::from(Self::from(self.0 * *rhs as f64)))
|
||||
self * *rhs as i64
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<i64> for Dollars {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: i64) -> Self::Output {
|
||||
Self::from(Cents::from(self) * rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<usize> for Dollars {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: usize) -> Self::Output {
|
||||
Self::from(Cents::from(self) * rhs)
|
||||
if self.is_nan() {
|
||||
self
|
||||
} else {
|
||||
Self::from(Cents::from(self) * rhs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,6 +240,12 @@ impl From<u128> for Dollars {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoredF64> for Dollars {
|
||||
fn from(value: StoredF64) -> Self {
|
||||
Self(*value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Close<Dollars>> for u128 {
|
||||
fn from(value: Close<Dollars>) -> Self {
|
||||
u128::from(*value)
|
||||
@@ -170,3 +257,80 @@ impl From<Dollars> for u128 {
|
||||
u128::from(Cents::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl AddAssign for Dollars {
|
||||
fn add_assign(&mut self, rhs: Self) {
|
||||
*self = Dollars::from(Cents::from(*self) + Cents::from(rhs));
|
||||
}
|
||||
}
|
||||
|
||||
impl CheckedSub for Dollars {
|
||||
fn checked_sub(self, rhs: Self) -> Option<Self> {
|
||||
if self.is_nan() {
|
||||
Some(self)
|
||||
} else {
|
||||
Cents::from(self)
|
||||
.checked_sub(Cents::from(rhs))
|
||||
.map(Dollars::from)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CheckedSub<usize> for Dollars {
|
||||
fn checked_sub(self, rhs: usize) -> Option<Self> {
|
||||
Some(Dollars::from(
|
||||
Cents::from(self).checked_sub(Cents::from(rhs)).unwrap(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for Dollars {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => true,
|
||||
(true, false) => false,
|
||||
(false, true) => false,
|
||||
(false, false) => self.0 == other.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for Dollars {}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl PartialOrd for Dollars {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for Dollars {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => Ordering::Equal,
|
||||
(true, false) => Ordering::Less,
|
||||
(false, true) => Ordering::Greater,
|
||||
(false, false) => self.0.partial_cmp(&other.0).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ByteView> for Dollars {
|
||||
fn from(value: ByteView) -> Self {
|
||||
let bytes = copy_first_8bytes(&value).unwrap();
|
||||
Self::from(f64::from_be_bytes(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Dollars> for ByteView {
|
||||
fn from(value: Dollars) -> Self {
|
||||
Self::from(&value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&Dollars> for ByteView {
|
||||
fn from(value: &Dollars) -> Self {
|
||||
Self::new(&value.to_be_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,14 @@
|
||||
use std::ops::{Add, Div};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
ops::{Add, Div},
|
||||
};
|
||||
|
||||
use serde::Serialize;
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use super::{Sats, StoredUsize};
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
Serialize,
|
||||
PartialEq,
|
||||
PartialOrd,
|
||||
FromBytes,
|
||||
Immutable,
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
)]
|
||||
#[derive(Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)]
|
||||
pub struct Feerate(f32);
|
||||
|
||||
impl From<(Sats, StoredUsize)> for Feerate {
|
||||
@@ -56,11 +48,34 @@ impl From<usize> for Feerate {
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for Feerate {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => true,
|
||||
(true, false) => false,
|
||||
(false, true) => false,
|
||||
(false, false) => self.0 == other.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for Feerate {}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for Feerate {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.0.partial_cmp(&other.0).unwrap()
|
||||
impl PartialOrd for Feerate {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for Feerate {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => Ordering::Equal,
|
||||
(true, false) => Ordering::Less,
|
||||
(false, true) => Ordering::Greater,
|
||||
(false, false) => self.0.partial_cmp(&other.0).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,12 @@ use super::Height;
|
||||
)]
|
||||
pub struct HalvingEpoch(u8);
|
||||
|
||||
impl HalvingEpoch {
|
||||
pub fn new(value: u8) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u8> for HalvingEpoch {
|
||||
fn from(value: u8) -> Self {
|
||||
Self(value)
|
||||
|
||||
@@ -4,12 +4,15 @@ use std::{
|
||||
};
|
||||
|
||||
use bitcoincore_rpc::{Client, RpcApi};
|
||||
use byteview::ByteView;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::CheckedSub;
|
||||
|
||||
use super::StoredUsize;
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Clone,
|
||||
@@ -147,11 +150,17 @@ impl From<u64> for Height {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoredUsize> for Height {
|
||||
fn from(value: StoredUsize) -> Self {
|
||||
Self(*value as u32)
|
||||
}
|
||||
}
|
||||
impl From<usize> for Height {
|
||||
fn from(value: usize) -> Self {
|
||||
Self(value as u32)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Height> for usize {
|
||||
fn from(value: Height) -> Self {
|
||||
value.0 as usize
|
||||
@@ -189,10 +198,9 @@ impl TryFrom<&std::path::Path> for Height {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<byteview::ByteView> for Height {
|
||||
type Error = crate::Error;
|
||||
fn try_from(value: byteview::ByteView) -> Result<Self, Self::Error> {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
impl From<ByteView> for Height {
|
||||
fn from(value: byteview::ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ mod txidprefix;
|
||||
mod txindex;
|
||||
mod txversion;
|
||||
mod unit;
|
||||
mod version;
|
||||
mod vin;
|
||||
mod vout;
|
||||
mod weekindex;
|
||||
@@ -80,6 +81,7 @@ pub use txidprefix::*;
|
||||
pub use txindex::*;
|
||||
pub use txversion::*;
|
||||
pub use unit::*;
|
||||
pub use version::*;
|
||||
pub use vin::*;
|
||||
pub use vout::*;
|
||||
pub use weekindex::*;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::ops::{Add, Div};
|
||||
|
||||
use derive_deref::Deref;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use serde::{Serialize, Serializer, ser::SerializeTuple};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
@@ -172,6 +172,7 @@ impl From<Close<Sats>> for OHLCSats {
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
@@ -259,6 +260,7 @@ where
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
@@ -346,6 +348,7 @@ where
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
@@ -433,6 +436,7 @@ where
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
|
||||
@@ -32,6 +32,46 @@ pub enum OutputType {
|
||||
Unknown = 255,
|
||||
}
|
||||
|
||||
impl OutputType {
|
||||
pub fn is_spendable(&self) -> bool {
|
||||
match self {
|
||||
Self::P2PK65 => true,
|
||||
Self::P2PK33 => true,
|
||||
Self::P2PKH => true,
|
||||
Self::P2MS => true,
|
||||
Self::P2SH => true,
|
||||
Self::OpReturn => false,
|
||||
Self::P2WPKH => true,
|
||||
Self::P2WSH => true,
|
||||
Self::P2TR => true,
|
||||
Self::P2A => true,
|
||||
Self::Empty => true,
|
||||
Self::Unknown => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_unspendable(&self) -> bool {
|
||||
!self.is_spendable()
|
||||
}
|
||||
|
||||
pub fn as_vec() -> Vec<Self> {
|
||||
vec![
|
||||
Self::P2PK65,
|
||||
Self::P2PK33,
|
||||
Self::P2PKH,
|
||||
Self::P2MS,
|
||||
Self::P2SH,
|
||||
Self::OpReturn,
|
||||
Self::P2WPKH,
|
||||
Self::P2WSH,
|
||||
Self::P2TR,
|
||||
Self::P2A,
|
||||
Self::Empty,
|
||||
Self::Unknown,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&ScriptBuf> for OutputType {
|
||||
fn from(script: &ScriptBuf) -> Self {
|
||||
if script.is_p2pk() {
|
||||
|
||||
@@ -6,7 +6,7 @@ use serde::Serialize;
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::{CheckedSub, Error};
|
||||
use crate::CheckedSub;
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
@@ -82,10 +82,9 @@ impl Add<OutputTypeIndex> for OutputTypeIndex {
|
||||
Self(self.0 + rhs.0)
|
||||
}
|
||||
}
|
||||
impl TryFrom<ByteView> for OutputTypeIndex {
|
||||
type Error = Error;
|
||||
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
impl From<ByteView> for OutputTypeIndex {
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
impl From<OutputTypeIndex> for ByteView {
|
||||
|
||||
@@ -3,11 +3,13 @@ use std::{
|
||||
ops::{Add, AddAssign, Div, Mul, SubAssign},
|
||||
};
|
||||
|
||||
use bincode::{Decode, Encode};
|
||||
use bitcoin::Amount;
|
||||
use serde::Serialize;
|
||||
use byteview::ByteView;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::CheckedSub;
|
||||
use crate::{CheckedSub, copy_first_8bytes};
|
||||
|
||||
use super::{Bitcoin, Cents, Dollars, Height};
|
||||
|
||||
@@ -25,17 +27,30 @@ use super::{Bitcoin, Cents, Dollars, Height};
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
Encode,
|
||||
Decode,
|
||||
)]
|
||||
pub struct Sats(u64);
|
||||
|
||||
#[allow(clippy::inconsistent_digit_grouping)]
|
||||
impl Sats {
|
||||
pub const ZERO: Self = Self(0);
|
||||
pub const MAX: Self = Self(u64::MAX);
|
||||
pub const ONE_BTC: Self = Self(100_000_000);
|
||||
pub const ONE_BTC: Self = Self(1_00_000_000);
|
||||
pub const FIFTY_BTC: Self = Self(50_00_000_000);
|
||||
|
||||
pub fn new(sats: u64) -> Self {
|
||||
Self(sats)
|
||||
}
|
||||
|
||||
pub fn is_zero(&self) -> bool {
|
||||
*self == Self::ZERO
|
||||
}
|
||||
|
||||
pub fn is_not_zero(&self) -> bool {
|
||||
*self != Self::ZERO
|
||||
}
|
||||
}
|
||||
|
||||
impl Add for Sats {
|
||||
@@ -57,6 +72,12 @@ impl CheckedSub<Sats> for Sats {
|
||||
}
|
||||
}
|
||||
|
||||
impl CheckedSub<usize> for Sats {
|
||||
fn checked_sub(self, rhs: usize) -> Option<Self> {
|
||||
self.0.checked_sub(rhs as u64).map(Self::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl SubAssign for Sats {
|
||||
fn sub_assign(&mut self, rhs: Self) {
|
||||
*self = self.checked_sub(rhs).unwrap();
|
||||
@@ -66,21 +87,28 @@ impl SubAssign for Sats {
|
||||
impl Mul<Sats> for Sats {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: Sats) -> Self::Output {
|
||||
Sats::from(self.0 * rhs.0)
|
||||
Sats::from(self.0.checked_mul(rhs.0).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<usize> for Sats {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: usize) -> Self::Output {
|
||||
Sats::from(self.0.checked_mul(rhs as u64).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<u64> for Sats {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: u64) -> Self::Output {
|
||||
Sats::from(self.0 * rhs)
|
||||
Sats::from(self.0.checked_mul(rhs).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<Height> for Sats {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: Height) -> Self::Output {
|
||||
Sats::from(self.0 * u64::from(rhs))
|
||||
Sats::from(self.0.checked_mul(u64::from(rhs)).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,6 +131,17 @@ impl Div<Dollars> for Sats {
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<Sats> for Sats {
|
||||
type Output = Self;
|
||||
fn div(self, rhs: Sats) -> Self::Output {
|
||||
if rhs.0 == 0 {
|
||||
Self(0)
|
||||
} else {
|
||||
Self(self.0 / rhs.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Div<usize> for Sats {
|
||||
type Output = Self;
|
||||
fn div(self, rhs: usize) -> Self::Output {
|
||||
@@ -134,6 +173,12 @@ impl From<Sats> for f64 {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Sats> for usize {
|
||||
fn from(value: Sats) -> Self {
|
||||
value.0 as usize
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Amount> for Sats {
|
||||
fn from(value: Amount) -> Self {
|
||||
Self(value.to_sat())
|
||||
@@ -171,3 +216,29 @@ impl From<Sats> for u128 {
|
||||
value.0 as u128
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ByteView> for Sats {
|
||||
fn from(value: ByteView) -> Self {
|
||||
let bytes = copy_first_8bytes(&value).unwrap();
|
||||
Self::from(u64::from_be_bytes(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&Sats> for ByteView {
|
||||
fn from(value: &Sats) -> Self {
|
||||
Self::new(&value.0.to_be_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Sats> for ByteView {
|
||||
fn from(value: Sats) -> Self {
|
||||
Self::from(&value)
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<Sats> for usize {
|
||||
type Output = Sats;
|
||||
fn mul(self, rhs: Sats) -> Self::Output {
|
||||
Self::Output::from(rhs.0 * self as u64)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
use std::ops::{Add, Div, Mul, Sub};
|
||||
use core::panic;
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
ops::{Add, Div, Mul, Sub},
|
||||
};
|
||||
|
||||
use derive_deref::Deref;
|
||||
use serde::Serialize;
|
||||
@@ -9,18 +13,7 @@ use crate::CheckedSub;
|
||||
use super::{Dollars, StoredF64};
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Deref,
|
||||
Default,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
PartialOrd,
|
||||
FromBytes,
|
||||
Immutable,
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Serialize,
|
||||
Debug, Deref, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
|
||||
)]
|
||||
pub struct StoredF32(f32);
|
||||
|
||||
@@ -32,10 +25,19 @@ impl From<f32> for StoredF32 {
|
||||
|
||||
impl From<f64> for StoredF32 {
|
||||
fn from(value: f64) -> Self {
|
||||
if value > f32::MAX as f64 {
|
||||
panic!("f64 is too big")
|
||||
}
|
||||
Self(value as f32)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoredF32> for f64 {
|
||||
fn from(value: StoredF32) -> Self {
|
||||
value.0 as f64
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoredF64> for StoredF32 {
|
||||
fn from(value: StoredF64) -> Self {
|
||||
Self(*value as f32)
|
||||
@@ -74,12 +76,9 @@ impl From<StoredF32> for f32 {
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for StoredF32 {}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for StoredF32 {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.0.partial_cmp(&other.0).unwrap()
|
||||
impl From<Dollars> for StoredF32 {
|
||||
fn from(value: Dollars) -> Self {
|
||||
StoredF32::from(f64::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,3 +116,35 @@ impl Sub<StoredF32> for StoredF32 {
|
||||
Self(self.0 - rhs.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for StoredF32 {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => true,
|
||||
(true, false) => false,
|
||||
(false, true) => false,
|
||||
(false, false) => self.0 == other.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for StoredF32 {}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl PartialOrd for StoredF32 {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for StoredF32 {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => Ordering::Equal,
|
||||
(true, false) => Ordering::Less,
|
||||
(false, true) => Ordering::Greater,
|
||||
(false, false) => self.0.partial_cmp(&other.0).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +1,24 @@
|
||||
use std::ops::{Add, Div, Mul};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
f64,
|
||||
ops::{Add, Div, Mul},
|
||||
};
|
||||
|
||||
use derive_deref::Deref;
|
||||
use serde::Serialize;
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::CheckedSub;
|
||||
use crate::{Bitcoin, CheckedSub, Dollars};
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Deref,
|
||||
Default,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
PartialOrd,
|
||||
FromBytes,
|
||||
Immutable,
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Serialize,
|
||||
Debug, Deref, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
|
||||
)]
|
||||
pub struct StoredF64(f64);
|
||||
|
||||
impl StoredF64 {
|
||||
pub const NAN: Self = Self(f64::NAN);
|
||||
}
|
||||
|
||||
impl From<f64> for StoredF64 {
|
||||
fn from(value: f64) -> Self {
|
||||
Self(value)
|
||||
@@ -67,12 +64,9 @@ impl From<StoredF64> for f64 {
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for StoredF64 {}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for StoredF64 {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.0.partial_cmp(&other.0).unwrap()
|
||||
impl From<Dollars> for StoredF64 {
|
||||
fn from(value: Dollars) -> Self {
|
||||
Self(f64::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,3 +75,41 @@ impl CheckedSub<usize> for StoredF64 {
|
||||
Some(Self(self.0 - rhs as f64))
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for StoredF64 {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => true,
|
||||
(true, false) => false,
|
||||
(false, true) => false,
|
||||
(false, false) => self.0 == other.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for StoredF64 {}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl PartialOrd for StoredF64 {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::derive_ord_xor_partial_ord)]
|
||||
impl Ord for StoredF64 {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
match (self.0.is_nan(), other.0.is_nan()) {
|
||||
(true, true) => Ordering::Equal,
|
||||
(true, false) => Ordering::Less,
|
||||
(false, true) => Ordering::Greater,
|
||||
(false, false) => self.0.partial_cmp(&other.0).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Bitcoin> for StoredF64 {
|
||||
fn from(value: Bitcoin) -> Self {
|
||||
Self(f64::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ use super::{
|
||||
Debug,
|
||||
Deref,
|
||||
Clone,
|
||||
Default,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::ops::{Add, Div};
|
||||
|
||||
use derive_deref::Deref;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use serde::Serialize;
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
@@ -15,6 +15,7 @@ use super::{
|
||||
#[derive(
|
||||
Debug,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Clone,
|
||||
Default,
|
||||
Copy,
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use std::ops::{Add, Div};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
ops::{Add, Div},
|
||||
};
|
||||
|
||||
use derive_deref::Deref;
|
||||
use jiff::{civil::date, tz::TimeZone};
|
||||
@@ -26,6 +29,8 @@ use super::Date;
|
||||
)]
|
||||
pub struct Timestamp(u32);
|
||||
|
||||
const ONE_DAY_IN_SEC: i64 = 24 * 60 * 60;
|
||||
|
||||
impl Timestamp {
|
||||
pub const ZERO: Self = Self(0);
|
||||
|
||||
@@ -34,10 +39,41 @@ impl Timestamp {
|
||||
}
|
||||
|
||||
pub fn floor_seconds(self) -> Self {
|
||||
let t = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC);
|
||||
let d = jiff::civil::DateTime::from(t);
|
||||
let d = date(d.year(), d.month(), d.day()).at(d.hour(), d.minute(), 0, 0);
|
||||
Self::from(d.to_zoned(TimeZone::UTC).unwrap().timestamp())
|
||||
let zoned = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC);
|
||||
let date_time = jiff::civil::DateTime::from(zoned);
|
||||
let trunc_date_time = date(date_time.year(), date_time.month(), date_time.day()).at(
|
||||
date_time.hour(),
|
||||
date_time.minute(),
|
||||
0,
|
||||
0,
|
||||
);
|
||||
Self::from(trunc_date_time.to_zoned(TimeZone::UTC).unwrap().timestamp())
|
||||
}
|
||||
|
||||
pub fn difference_in_days_between(&self, other: Self) -> usize {
|
||||
match self.cmp(&other) {
|
||||
Ordering::Equal => 0,
|
||||
Ordering::Greater => other.difference_in_days_between(*self),
|
||||
Ordering::Less => {
|
||||
(jiff::Timestamp::from(*self)
|
||||
.duration_until(jiff::Timestamp::from(other))
|
||||
.as_secs()
|
||||
/ ONE_DAY_IN_SEC) as usize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn difference_in_days_between_float(&self, other: Self) -> f64 {
|
||||
match self.cmp(&other) {
|
||||
Ordering::Equal => 0.0,
|
||||
Ordering::Greater => other.difference_in_days_between_float(*self),
|
||||
Ordering::Less => {
|
||||
jiff::Timestamp::from(*self)
|
||||
.duration_until(jiff::Timestamp::from(other))
|
||||
.as_secs() as f64
|
||||
/ ONE_DAY_IN_SEC as f64
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ use derive_deref::Deref;
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::{Error, copy_first_8bytes};
|
||||
use crate::copy_first_8bytes;
|
||||
|
||||
use super::Txid;
|
||||
|
||||
@@ -35,10 +35,9 @@ impl From<&Txid> for TxidPrefix {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<ByteView> for TxidPrefix {
|
||||
type Error = Error;
|
||||
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
impl From<ByteView> for TxidPrefix {
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use serde::Serialize;
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::{CheckedSub, Error};
|
||||
use crate::CheckedSub;
|
||||
|
||||
use super::StoredU32;
|
||||
|
||||
@@ -95,10 +95,9 @@ impl From<TxIndex> for usize {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<ByteView> for TxIndex {
|
||||
type Error = Error;
|
||||
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
|
||||
Ok(Self::read_from_bytes(&value)?)
|
||||
impl From<ByteView> for TxIndex {
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
impl From<TxIndex> for ByteView {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::{
|
||||
fs,
|
||||
io::{self, Read},
|
||||
iter::Sum,
|
||||
ops::Add,
|
||||
path::Path,
|
||||
};
|
||||
@@ -13,14 +14,14 @@ use crate::{Error, Result};
|
||||
#[derive(
|
||||
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout,
|
||||
)]
|
||||
pub struct Version(u32);
|
||||
pub struct Version(u64);
|
||||
|
||||
impl Version {
|
||||
pub const ZERO: Self = Self(0);
|
||||
pub const ONE: Self = Self(1);
|
||||
pub const TWO: Self = Self(2);
|
||||
|
||||
pub const fn new(v: u32) -> Self {
|
||||
pub const fn new(v: u64) -> Self {
|
||||
Self(v)
|
||||
}
|
||||
|
||||
@@ -32,7 +33,9 @@ impl Version {
|
||||
Self(self.0.swap_bytes())
|
||||
}
|
||||
|
||||
pub fn validate(&self, path: &Path) -> Result<()> {
|
||||
/// Ok(true) if existed and is same
|
||||
/// Ok(false) if didn't exist
|
||||
pub fn validate(&self, path: &Path) -> Result<bool> {
|
||||
if let Ok(prev_version) = Version::try_from(path) {
|
||||
if prev_version != *self {
|
||||
if prev_version.swap_bytes() == *self {
|
||||
@@ -43,14 +46,16 @@ impl Version {
|
||||
expected: *self,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u32> for Version {
|
||||
fn from(value: u32) -> Self {
|
||||
impl From<u64> for Version {
|
||||
fn from(value: u64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
@@ -58,7 +63,7 @@ impl From<u32> for Version {
|
||||
impl TryFrom<&Path> for Version {
|
||||
type Error = Error;
|
||||
fn try_from(value: &Path) -> Result<Self, Self::Error> {
|
||||
let mut buf = [0; 4];
|
||||
let mut buf = [0; 8];
|
||||
fs::read(value)?.as_slice().read_exact(&mut buf)?;
|
||||
Ok(*(Self::ref_from_bytes(&buf)?))
|
||||
}
|
||||
@@ -70,3 +75,9 @@ impl Add<Version> for Version {
|
||||
Self(self.0 + rhs.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Sum for Version {
|
||||
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
|
||||
iter.fold(Self::ZERO, Add::add)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
use crate::{Error, Result};
|
||||
|
||||
#[allow(clippy::result_unit_err)]
|
||||
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8], ()> {
|
||||
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
|
||||
let mut buf: [u8; 8] = [0; 8];
|
||||
let buf_len = buf.len();
|
||||
if slice.len() < buf_len {
|
||||
return Err(());
|
||||
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
|
||||
}
|
||||
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
|
||||
buf[i] = *r;
|
||||
|
||||
@@ -3,9 +3,11 @@ mod checked_sub;
|
||||
mod paths;
|
||||
mod pause;
|
||||
mod rlimit;
|
||||
mod serde;
|
||||
|
||||
pub use bytes::*;
|
||||
pub use checked_sub::*;
|
||||
pub use paths::*;
|
||||
pub use pause::*;
|
||||
pub use rlimit::*;
|
||||
pub use serde::*;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
pub fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
T: Deserialize<'de> + Default,
|
||||
{
|
||||
match T::deserialize(deserializer) {
|
||||
Ok(v) => Ok(v),
|
||||
Err(_) => Ok(T::default()),
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ description = "An exit blocker built on top of ctrlc"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://kibo.money">
|
||||
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
|
||||
@@ -57,6 +57,10 @@ impl Exit {
|
||||
self.blocking.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn blocked(&self) -> bool {
|
||||
self.blocking.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn release(&self) {
|
||||
self.blocking.store(false, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ description = "A Bitcoin price fetcher"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://kibo.money">
|
||||
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
@@ -34,4 +31,4 @@
|
||||
</a>
|
||||
</p>
|
||||
|
||||
A crate that can fetch the Bitcoin price, either by date or height, from multiple APIs such Kraken, Binance and Kibo.money.
|
||||
A crate that can fetch the Bitcoin price, either by date or height, from Binance and Kibo.
|
||||
|
||||
@@ -1,18 +1,35 @@
|
||||
use brk_core::Date;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_core::{Date, Height};
|
||||
use brk_fetcher::{BRK, Binance, Fetcher, Kraken};
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
brk_logger::init(None);
|
||||
|
||||
let mut brk = BRK::default();
|
||||
dbg!(brk.get_from_height(Height::new(900_000))?);
|
||||
dbg!(brk.get_from_date(Date::new(2025, 6, 7))?);
|
||||
|
||||
let mut fetcher = Fetcher::import(None)?;
|
||||
|
||||
dbg!(fetcher.get_date(Date::new(2025, 1, 1))?);
|
||||
Binance::fetch_1d().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
Kraken::fetch_1d().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
Binance::fetch_1mn().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
Kraken::fetch_1mn().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
|
||||
dbg!(fetcher.get_date(Date::new(2025, 6, 5))?);
|
||||
dbg!(fetcher.get_height(
|
||||
881_000_u32.into(),
|
||||
1740683986_u32.into(),
|
||||
Some(1740683000_u32.into())
|
||||
899911_u32.into(),
|
||||
1749133056_u32.into(),
|
||||
Some(1749132055_u32.into())
|
||||
)?);
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, fetchers::retry};
|
||||
pub struct Binance {
|
||||
path: Option<PathBuf>,
|
||||
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
|
||||
_1d: Option<BTreeMap<Date, OHLCCents>>,
|
||||
pub _1d: Option<BTreeMap<Date, OHLCCents>>,
|
||||
har: Option<BTreeMap<Timestamp, OHLCCents>>,
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ impl Binance {
|
||||
}
|
||||
|
||||
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
|
||||
info!("Fetching daily prices from Kraken...");
|
||||
info!("Fetching daily prices from Binance...");
|
||||
|
||||
retry(
|
||||
|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?),
|
||||
@@ -238,4 +238,9 @@ impl Binance {
|
||||
fn url(query: &str) -> String {
|
||||
format!("https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&{query}")
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self._1d.take();
|
||||
self._1mn.take();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use brk_core::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
|
||||
use color_eyre::eyre::{ContextCompat, eyre};
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{Close, Dollars, High, Low, Open, fetchers::retry};
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct BRK {
|
||||
height_to_ohlc: BTreeMap<Height, Vec<OHLCCents>>,
|
||||
dateindex_to_ohlc: BTreeMap<DateIndex, Vec<OHLCCents>>,
|
||||
}
|
||||
|
||||
const API_URL: &str = "https://bitcoinresearchkit.org/api";
|
||||
|
||||
const RETRIES: usize = 10;
|
||||
|
||||
const CHUNK_SIZE: usize = 10_000;
|
||||
|
||||
impl BRK {
|
||||
pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
|
||||
let key = height.checked_sub(height % CHUNK_SIZE).unwrap();
|
||||
|
||||
#[allow(clippy::map_entry)]
|
||||
if !self.height_to_ohlc.contains_key(&key)
|
||||
|| ((key + self.height_to_ohlc.get(&key).unwrap().len()) <= height)
|
||||
{
|
||||
self.height_to_ohlc.insert(
|
||||
key,
|
||||
Self::fetch_height_prices(key).inspect_err(|e| {
|
||||
dbg!(e);
|
||||
})?,
|
||||
);
|
||||
}
|
||||
|
||||
self.height_to_ohlc
|
||||
.get(&key)
|
||||
.unwrap()
|
||||
.get(usize::from(height.checked_sub(key).unwrap()))
|
||||
.cloned()
|
||||
.ok_or(eyre!("Couldn't find height in kibo"))
|
||||
}
|
||||
|
||||
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
|
||||
info!("Fetching Kibo height {height} prices...");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
let url = format!(
|
||||
"{API_URL}/query?index=height&values=ohlc&from={}&to={}",
|
||||
height,
|
||||
height + CHUNK_SIZE
|
||||
);
|
||||
|
||||
let body: Value = minreq::get(url).send()?.json()?;
|
||||
|
||||
body.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(Self::value_to_ohlc)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_from_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
|
||||
let dateindex = DateIndex::try_from(date)?;
|
||||
|
||||
let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap();
|
||||
|
||||
#[allow(clippy::map_entry)]
|
||||
if !self.dateindex_to_ohlc.contains_key(&key)
|
||||
|| ((key + self.dateindex_to_ohlc.get(&key).unwrap().len()) <= dateindex)
|
||||
{
|
||||
self.dateindex_to_ohlc.insert(
|
||||
key,
|
||||
Self::fetch_date_prices(key).inspect_err(|e| {
|
||||
dbg!(e);
|
||||
})?,
|
||||
);
|
||||
}
|
||||
|
||||
self.dateindex_to_ohlc
|
||||
.get(&key)
|
||||
.unwrap()
|
||||
.get(usize::from(dateindex.checked_sub(key).unwrap()))
|
||||
.cloned()
|
||||
.ok_or(eyre!("Couldn't find date in kibo"))
|
||||
}
|
||||
|
||||
fn fetch_date_prices(dateindex: DateIndex) -> color_eyre::Result<Vec<OHLCCents>> {
|
||||
info!("Fetching Kibo dateindex {dateindex} prices...");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
let url = format!(
|
||||
"{API_URL}/query?index=dateindex&values=ohlc&from={}&to={}",
|
||||
dateindex,
|
||||
dateindex + CHUNK_SIZE
|
||||
);
|
||||
|
||||
let body: Value = minreq::get(url).send()?.json()?;
|
||||
|
||||
body.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(Self::value_to_ohlc)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
|
||||
let ohlc = value.as_array().context("Expect as_array to work")?;
|
||||
|
||||
let get_value = |index: usize| -> color_eyre::Result<_> {
|
||||
Ok(Cents::from(Dollars::from(
|
||||
ohlc.get(index)
|
||||
.context("Expect index key to work")?
|
||||
.as_f64()
|
||||
.context("Expect as_f64 to work")?,
|
||||
)))
|
||||
};
|
||||
|
||||
Ok(OHLCCents::from((
|
||||
Open::new(get_value(0)?),
|
||||
High::new(get_value(1)?),
|
||||
Low::new(get_value(2)?),
|
||||
Close::new(get_value(3)?),
|
||||
)))
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.height_to_ohlc.clear();
|
||||
self.dateindex_to_ohlc.clear();
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
use std::{collections::BTreeMap, str::FromStr};
|
||||
|
||||
use brk_core::{CheckedSub, Date, Height, OHLCCents};
|
||||
use color_eyre::eyre::{ContextCompat, eyre};
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{Cents, Close, Dollars, High, Low, Open, fetchers::retry};
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct Kibo {
|
||||
height_to_ohlc_vec: BTreeMap<Height, Vec<OHLCCents>>,
|
||||
year_to_date_to_ohlc: BTreeMap<u16, BTreeMap<Date, OHLCCents>>,
|
||||
}
|
||||
|
||||
const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
|
||||
|
||||
const RETRIES: usize = 10;
|
||||
|
||||
impl Kibo {
|
||||
pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
|
||||
let key = height.checked_sub(height % 10_000).unwrap_or_default();
|
||||
|
||||
#[allow(clippy::map_entry)]
|
||||
if !self.height_to_ohlc_vec.contains_key(&key)
|
||||
|| ((key + self.height_to_ohlc_vec.get(&key).unwrap().len()) <= height)
|
||||
{
|
||||
self.height_to_ohlc_vec.insert(
|
||||
key,
|
||||
Self::fetch_height_prices(key).inspect_err(|e| {
|
||||
dbg!(e);
|
||||
})?,
|
||||
);
|
||||
}
|
||||
|
||||
self.height_to_ohlc_vec
|
||||
.get(&key)
|
||||
.unwrap()
|
||||
.get(usize::from(height.checked_sub(key).unwrap()))
|
||||
.cloned()
|
||||
.ok_or(eyre!("Couldn't find height in kibo"))
|
||||
}
|
||||
|
||||
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
|
||||
info!("Fetching Kibo height {height} prices...");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
let url = format!("{KIBO_OFFICIAL_URL}/height-to-price?chunk={}", height);
|
||||
|
||||
let body: Value = minreq::get(url).send()?.json()?;
|
||||
|
||||
body.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("dataset")
|
||||
.context("Expect object to have dataset")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("map")
|
||||
.context("Expect to have map")?
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(Self::value_to_ohlc)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_from_date(&mut self, date: &Date) -> color_eyre::Result<OHLCCents> {
|
||||
let year = date.year();
|
||||
|
||||
#[allow(clippy::map_entry)]
|
||||
if !self.year_to_date_to_ohlc.contains_key(&year)
|
||||
|| self
|
||||
.year_to_date_to_ohlc
|
||||
.get(&year)
|
||||
.unwrap()
|
||||
.last_key_value()
|
||||
.unwrap()
|
||||
.0
|
||||
<= date
|
||||
{
|
||||
self.year_to_date_to_ohlc
|
||||
.insert(year, Self::fetch_date_prices(year)?);
|
||||
}
|
||||
|
||||
self.year_to_date_to_ohlc
|
||||
.get(&year)
|
||||
.unwrap()
|
||||
.get(date)
|
||||
.cloned()
|
||||
.ok_or(eyre!("Couldn't find date in kibo"))
|
||||
}
|
||||
|
||||
fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
|
||||
info!("Fetching Kibo date {year} prices...");
|
||||
|
||||
retry(
|
||||
|_| {
|
||||
let body: Value =
|
||||
minreq::get(format!("{KIBO_OFFICIAL_URL}/date-to-price?chunk={}", year))
|
||||
.send()?
|
||||
.json()?;
|
||||
|
||||
body.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("dataset")
|
||||
.context("Expect object to have dataset")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("map")
|
||||
.context("Expect to have map")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.iter()
|
||||
.map(|(serialized_date, value)| -> color_eyre::Result<_> {
|
||||
let date =
|
||||
Date::from(jiff::civil::Date::from_str(serialized_date).unwrap());
|
||||
Ok((date, Self::value_to_ohlc(value)?))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
|
||||
let ohlc = value.as_object().context("Expect as_object to work")?;
|
||||
|
||||
let get_value = |key: &str| -> color_eyre::Result<_> {
|
||||
Ok(Cents::from(Dollars::from(
|
||||
ohlc.get(key)
|
||||
.context("Expect get key to work")?
|
||||
.as_f64()
|
||||
.context("Expect as_f64 to work")?,
|
||||
)))
|
||||
};
|
||||
|
||||
Ok(OHLCCents::from((
|
||||
Open::new(get_value("open")?),
|
||||
High::new(get_value("high")?),
|
||||
Low::new(get_value("low")?),
|
||||
Close::new(get_value("close")?),
|
||||
)))
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@ impl Kraken {
|
||||
)
|
||||
}
|
||||
|
||||
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
|
||||
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
|
||||
info!("Fetching 1mn prices from Kraken...");
|
||||
|
||||
retry(
|
||||
@@ -54,7 +54,7 @@ impl Kraken {
|
||||
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
|
||||
}
|
||||
|
||||
fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
|
||||
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
|
||||
info!("Fetching daily prices from Kraken...");
|
||||
|
||||
retry(
|
||||
@@ -129,4 +129,9 @@ impl Kraken {
|
||||
fn url(interval: usize) -> String {
|
||||
format!("https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval={interval}")
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self._1d.take();
|
||||
self._1mn.take();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
mod binance;
|
||||
mod kibo;
|
||||
mod brk;
|
||||
mod kraken;
|
||||
mod retry;
|
||||
|
||||
pub use binance::*;
|
||||
pub use kibo::*;
|
||||
pub use brk::*;
|
||||
pub use kraken::*;
|
||||
use retry::*;
|
||||
|
||||
@@ -3,20 +3,23 @@
|
||||
#![doc = include_str!("../examples/main.rs")]
|
||||
#![doc = "```"]
|
||||
|
||||
use std::{collections::BTreeMap, fs, path::Path};
|
||||
use std::{collections::BTreeMap, fs, path::Path, thread::sleep, time::Duration};
|
||||
|
||||
use brk_core::{Cents, Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
|
||||
use brk_core::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
|
||||
use color_eyre::eyre::Error;
|
||||
|
||||
mod fetchers;
|
||||
|
||||
use fetchers::*;
|
||||
pub use fetchers::*;
|
||||
use log::info;
|
||||
|
||||
const TRIES: usize = 12 * 60;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Fetcher {
|
||||
binance: Binance,
|
||||
kraken: Kraken,
|
||||
kibo: Kibo,
|
||||
brk: BRK,
|
||||
}
|
||||
|
||||
impl Fetcher {
|
||||
@@ -28,20 +31,37 @@ impl Fetcher {
|
||||
Ok(Self {
|
||||
binance: Binance::init(hars_path),
|
||||
kraken: Kraken::default(),
|
||||
kibo: Kibo::default(),
|
||||
brk: BRK::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
|
||||
self.get_date_(date, 0)
|
||||
}
|
||||
|
||||
fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
|
||||
self.kraken
|
||||
.get_from_1d(&date)
|
||||
.or_else(|e| {
|
||||
eprintln!("{e}");
|
||||
.or_else(|_| {
|
||||
// eprintln!("{e}");
|
||||
self.binance.get_from_1d(&date)
|
||||
})
|
||||
.or_else(|_| {
|
||||
// eprintln!("{e}");
|
||||
self.brk.get_from_date(date)
|
||||
})
|
||||
.or_else(|e| {
|
||||
eprintln!("{e}");
|
||||
self.kibo.get_from_date(&date)
|
||||
sleep(Duration::from_secs(60));
|
||||
|
||||
if tries < TRIES {
|
||||
self.clear();
|
||||
// dbg!(e, date, &self.binance._1d);
|
||||
info!("Retrying to fetch date price...");
|
||||
self.get_date_(date, tries + 1)
|
||||
} else {
|
||||
info!("Failed to fetch date prices...");
|
||||
Err(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -50,6 +70,16 @@ impl Fetcher {
|
||||
height: Height,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
) -> color_eyre::Result<OHLCCents> {
|
||||
self.get_height_(height, timestamp, previous_timestamp, 0)
|
||||
}
|
||||
|
||||
fn get_height_(
|
||||
&mut self,
|
||||
height: Height,
|
||||
timestamp: Timestamp,
|
||||
previous_timestamp: Option<Timestamp>,
|
||||
tries: usize,
|
||||
) -> color_eyre::Result<OHLCCents> {
|
||||
let timestamp = timestamp.floor_seconds();
|
||||
|
||||
@@ -62,15 +92,32 @@ impl Fetcher {
|
||||
let ohlc = self
|
||||
.kraken
|
||||
.get_from_1mn(timestamp, previous_timestamp)
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("{e}");
|
||||
.unwrap_or_else(|_report| {
|
||||
// eprintln!("{_report}");
|
||||
self.binance
|
||||
.get_from_1mn(timestamp, previous_timestamp)
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("{e}");
|
||||
self.kibo.get_from_height(height).unwrap_or_else(|e| {
|
||||
.unwrap_or_else(|_report| {
|
||||
// // eprintln!("{_report}");
|
||||
self.brk.get_from_height(height).unwrap_or_else(|_report| {
|
||||
// eprintln!("{_report}");
|
||||
|
||||
sleep(Duration::from_secs(60));
|
||||
|
||||
if tries < TRIES {
|
||||
self.clear();
|
||||
|
||||
info!("Retrying to fetch height prices...");
|
||||
// dbg!((height, timestamp, previous_timestamp));
|
||||
|
||||
return self
|
||||
.get_height_(height, timestamp, previous_timestamp, tries + 1)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
info!("Failed to fetch height prices");
|
||||
|
||||
let date = Date::from(timestamp);
|
||||
eprintln!("{e}");
|
||||
// eprintln!("{e}");
|
||||
panic!(
|
||||
"
|
||||
Can't find the price for: height: {height} - date: {date}
|
||||
@@ -92,8 +139,6 @@ How to fix this:
|
||||
})
|
||||
});
|
||||
|
||||
// self.ohlc.height.insert(height, ohlc);
|
||||
|
||||
Ok(ohlc)
|
||||
}
|
||||
|
||||
@@ -138,4 +183,10 @@ How to fix this:
|
||||
|
||||
Ok(final_ohlc)
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.kraken.clear();
|
||||
self.binance.clear();
|
||||
self.brk.clear();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ description = "A Bitcoin Core indexer built on top of brk_parser"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
@@ -11,12 +12,11 @@ bitcoin = { workspace = true }
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_store = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
byteview = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
fjall = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
zerocopy = { workspace = true }
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://kibo.money">
|
||||
<img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::{path::Path, time::Instant};
|
||||
use brk_core::default_bitcoin_path;
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::{Parser, rpc};
|
||||
use brk_parser::Parser;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
@@ -14,9 +14,9 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let bitcoin_dir = default_bitcoin_path();
|
||||
|
||||
let rpc = Box::leak(Box::new(rpc::Client::new(
|
||||
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
|
||||
"http://localhost:8332",
|
||||
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
|
||||
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
|
||||
)?));
|
||||
let exit = Exit::new();
|
||||
|
||||
@@ -24,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let outputs = Path::new("../../_outputs");
|
||||
|
||||
let mut indexer = Indexer::new(outputs, false, false)?;
|
||||
let mut indexer = Indexer::new(outputs, false)?;
|
||||
|
||||
indexer.import_stores()?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
@@ -2,7 +2,7 @@ use bitcoincore_rpc::Client;
|
||||
use brk_core::{
|
||||
BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
|
||||
OutputType, OutputTypeIndex, P2AIndex, P2MSIndex, P2PK33Index, P2PK65Index, P2PKHIndex,
|
||||
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
|
||||
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, Result, TxIndex, UnknownOutputIndex,
|
||||
};
|
||||
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
|
||||
use brk_vec::{AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType};
|
||||
@@ -48,7 +48,7 @@ impl Indexes {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> {
|
||||
pub fn push_if_needed(&self, vecs: &mut Vecs) -> Result<()> {
|
||||
let height = self.height;
|
||||
vecs.height_to_first_txindex
|
||||
.push_if_needed(height, self.txindex)?;
|
||||
@@ -110,13 +110,7 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
|
||||
vecs.height_to_blockhash
|
||||
.iter()
|
||||
.get(*height)
|
||||
.is_none_or(|saved_blockhash| {
|
||||
let b = &rpc_blockhash != saved_blockhash.as_ref();
|
||||
if b {
|
||||
dbg!(rpc_blockhash, saved_blockhash.as_ref());
|
||||
}
|
||||
b
|
||||
})
|
||||
.is_none_or(|saved_blockhash| &rpc_blockhash != saved_blockhash.as_ref())
|
||||
})
|
||||
.unwrap_or(starting_height);
|
||||
|
||||
|
||||
@@ -12,13 +12,14 @@ use std::{
|
||||
|
||||
use brk_core::{
|
||||
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
|
||||
OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Vin, Vout, setrlimit,
|
||||
OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Version, Vin, Vout,
|
||||
setrlimit,
|
||||
};
|
||||
pub use brk_parser::*;
|
||||
|
||||
use bitcoin::{Transaction, TxIn, TxOut};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{AnyVec, Compressed, VecIterator};
|
||||
use brk_parser::Parser;
|
||||
use brk_vec::{AnyVec, VecIterator};
|
||||
use color_eyre::eyre::{ContextCompat, eyre};
|
||||
use fjall::TransactionalKeyspace;
|
||||
use log::{error, info};
|
||||
@@ -33,6 +34,7 @@ pub use vecs::*;
|
||||
|
||||
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
|
||||
const COLLISIONS_CHECKED_UP_TO: u32 = 893_000;
|
||||
const VERSION: Version = Version::ONE;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Indexer {
|
||||
@@ -40,21 +42,15 @@ pub struct Indexer {
|
||||
vecs: Option<Vecs>,
|
||||
stores: Option<Stores>,
|
||||
check_collisions: bool,
|
||||
compressed: Compressed,
|
||||
}
|
||||
|
||||
impl Indexer {
|
||||
pub fn new(
|
||||
outputs_dir: &Path,
|
||||
compressed: bool,
|
||||
check_collisions: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
pub fn new(outputs_dir: &Path, check_collisions: bool) -> color_eyre::Result<Self> {
|
||||
setrlimit()?;
|
||||
Ok(Self {
|
||||
path: outputs_dir.to_owned(),
|
||||
vecs: None,
|
||||
stores: None,
|
||||
compressed: Compressed::from(compressed),
|
||||
check_collisions,
|
||||
})
|
||||
}
|
||||
@@ -62,7 +58,7 @@ impl Indexer {
|
||||
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
|
||||
self.vecs = Some(Vecs::forced_import(
|
||||
&self.path.join("vecs/indexed"),
|
||||
self.compressed,
|
||||
VERSION + Version::ZERO,
|
||||
)?);
|
||||
Ok(())
|
||||
}
|
||||
@@ -70,14 +66,17 @@ impl Indexer {
|
||||
/// Do NOT import multiple times are things will break !!!
|
||||
/// Clone struct instead
|
||||
pub fn import_stores(&mut self) -> color_eyre::Result<()> {
|
||||
self.stores = Some(Stores::forced_import(&self.path.join("stores"))?);
|
||||
self.stores = Some(Stores::forced_import(
|
||||
&self.path.join("stores"),
|
||||
VERSION + Version::ZERO,
|
||||
)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn index(
|
||||
&mut self,
|
||||
parser: &Parser,
|
||||
rpc: &'static rpc::Client,
|
||||
rpc: &'static bitcoincore_rpc::Client,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<Indexes> {
|
||||
let starting_indexes = Indexes::try_from((
|
||||
@@ -114,6 +113,7 @@ impl Indexer {
|
||||
if starting_indexes.height > Height::try_from(rpc)?
|
||||
|| end.is_some_and(|end| starting_indexes.height > end)
|
||||
{
|
||||
info!("Up to date, nothing to index.");
|
||||
return Ok(starting_indexes);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,20 +1,15 @@
|
||||
use std::{fs, path::Path, thread};
|
||||
|
||||
use brk_core::{
|
||||
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex,
|
||||
TxidPrefix,
|
||||
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, Result,
|
||||
TxIndex, TxidPrefix, Value, Version,
|
||||
};
|
||||
use brk_vec::{AnyIterableVec, Value, Version};
|
||||
use brk_store::Store;
|
||||
use brk_vec::AnyIterableVec;
|
||||
use fjall::{PersistMode, TransactionalKeyspace};
|
||||
|
||||
use crate::Indexes;
|
||||
|
||||
mod base;
|
||||
mod meta;
|
||||
|
||||
pub use base::*;
|
||||
pub use meta::*;
|
||||
|
||||
use super::Vecs;
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -25,41 +20,46 @@ pub struct Stores {
|
||||
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl Stores {
|
||||
pub fn forced_import(path: &Path) -> color_eyre::Result<Self> {
|
||||
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
let keyspace = match Self::open_keyspace(path) {
|
||||
let keyspace = match brk_store::open_keyspace(path) {
|
||||
Ok(keyspace) => keyspace,
|
||||
Err(_) => {
|
||||
fs::remove_dir_all(path)?;
|
||||
return Self::forced_import(path);
|
||||
return Self::forced_import(path, version);
|
||||
}
|
||||
};
|
||||
|
||||
thread::scope(|scope| {
|
||||
let addressbyteshash_to_outputtypeindex = scope.spawn(|| {
|
||||
Store::import(
|
||||
keyspace.clone(),
|
||||
&keyspace,
|
||||
path,
|
||||
"addressbyteshash_to_outputtypeindex",
|
||||
Version::ZERO,
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
});
|
||||
let blockhashprefix_to_height = scope.spawn(|| {
|
||||
Store::import(
|
||||
keyspace.clone(),
|
||||
&keyspace,
|
||||
path,
|
||||
"blockhashprefix_to_height",
|
||||
Version::ZERO,
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
});
|
||||
let txidprefix_to_txindex = scope.spawn(|| {
|
||||
Store::import(
|
||||
keyspace.clone(),
|
||||
&keyspace,
|
||||
path,
|
||||
"txidprefix_to_txindex",
|
||||
Version::ZERO,
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -288,8 +288,8 @@ impl Stores {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
|
||||
thread::scope(|scope| -> fjall::Result<()> {
|
||||
pub fn commit(&mut self, height: Height) -> Result<()> {
|
||||
thread::scope(|scope| -> Result<()> {
|
||||
let addressbyteshash_to_outputtypeindex_commit_handle =
|
||||
scope.spawn(|| self.addressbyteshash_to_outputtypeindex.commit(height));
|
||||
let blockhashprefix_to_height_commit_handle =
|
||||
@@ -306,7 +306,9 @@ impl Stores {
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.keyspace.persist(PersistMode::SyncAll)
|
||||
self.keyspace
|
||||
.persist(PersistMode::SyncAll)
|
||||
.map_err(|e| e.into())
|
||||
}
|
||||
|
||||
pub fn rotate_memtables(&self) {
|
||||
@@ -314,10 +316,4 @@ impl Stores {
|
||||
self.blockhashprefix_to_height.rotate_memtable();
|
||||
self.txidprefix_to_txindex.rotate_memtable();
|
||||
}
|
||||
|
||||
fn open_keyspace(path: &Path) -> fjall::Result<TransactionalKeyspace> {
|
||||
fjall::Config::new(path.join("fjall"))
|
||||
.max_write_buffer_size(32 * 1024 * 1024)
|
||||
.open_transactional()
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user