Compare commits

...

66 Commits

Author SHA1 Message Date
nym21 a61926988a release: v0.0.88 2025-08-10 14:16:31 +02:00
nym21 bd8c4dfb6b website: fix options 2025-08-10 14:16:13 +02:00
nym21 ce9b4bc4dd release: v0.0.87 2025-08-10 13:43:36 +02:00
nym21 8b12b00114 cargo: update deps 2025-08-10 13:43:17 +02:00
nym21 1775cc1d54 release: v0.0.86 2025-08-10 13:20:04 +02:00
nym21 e4bd09df24 lock: update crates 2025-08-10 13:19:52 +02:00
nym21 5e8c7da4df global: convert brk_vecs to its own crates and repo (seqdb/vecdb) + changes 2025-08-10 12:49:41 +02:00
nym21 c85592eefe release: v0.0.85 2025-08-07 22:35:09 +02:00
nym21 05861c9113 mcp: upgrade + made stateless 2025-08-07 22:34:46 +02:00
nym21 3508d1e315 release: v0.0.84 2025-08-07 21:23:19 +02:00
nym21 e3177b8054 global: per crate build.rs 2025-08-07 21:22:38 +02:00
nym21 03e3760152 global: snapshot + lock file + better errors 2025-08-07 17:29:30 +02:00
nym21 4740610923 global: compressed vecs work again 2025-08-05 23:38:43 +02:00
nym21 e28a0cde55 vecs: fix race condition 2025-08-04 23:48:20 +02:00
nym21 5b855fd835 global: snapshot 2025-08-04 11:38:46 +02:00
nym21 a2f5704581 global: snapshot 2025-08-03 23:38:58 +02:00
nym21 f7aa9424db global: one big snapshot 2025-08-02 16:59:22 +02:00
nym21 aa8b47a3dd global: cleanup 2025-07-29 17:46:30 +02:00
nym21 11911c1898 release: v0.0.83 2025-07-26 23:35:43 +02:00
nym21 4814c1971d vecs: add linux punch hole impl 2025-07-26 23:35:23 +02:00
nym21 be9569f3fb release: v0.0.82 2025-07-26 22:09:21 +02:00
nym21 900e72f95a cargo: cleanup deps 2025-07-26 14:28:26 +02:00
nym21 d2827f188b computer: temp remove rayon 2025-07-26 14:24:06 +02:00
nym21 cf9903b759 computer: init file with min length and regions 2025-07-26 08:57:13 +02:00
nym21 23f96461f4 computer: remove libc dep 2025-07-26 08:42:33 +02:00
nym21 9f2fd26e98 computer: fixes 2025-07-26 08:41:19 +02:00
nym21 78d837c080 computer: flush + punch 2025-07-26 01:04:36 +02:00
nym21 241b9312b7 cli: config changes 2025-07-26 00:46:35 +02:00
nym21 ed70ad7378 indexer: take readers before last export 2025-07-25 22:45:41 +02:00
nym21 00213176d8 indexer: small changes 2025-07-25 22:38:15 +02:00
nym21 406650a45a vec: removed 2025-07-25 20:38:57 +02:00
nym21 56750ccf3c vecs: part 11 2025-07-25 20:27:15 +02:00
nym21 dfc286b393 vecs: part 10 2025-07-25 20:22:54 +02:00
nym21 49a66f72fc crates: update rapidhash 2025-07-24 17:32:38 +02:00
nym21 3f237689da vecs: part 9 2025-07-24 17:19:05 +02:00
nym21 cf1fb483b3 vecs: part 8 2025-07-24 16:48:50 +02:00
nym21 b10f5e3f67 vecs: part 7 2025-07-23 23:55:13 +02:00
nym21 c4fc24c513 vecs: part 6 2025-07-23 09:17:26 +02:00
nym21 3ac9c2d95e vecs: part 5 2025-07-22 21:26:50 +02:00
nym21 e5ab4dafc0 vecs: part 4 2025-07-22 17:36:34 +02:00
nym21 10ae1911c3 vecs: part 3 2025-07-22 15:10:07 +02:00
nym21 73ebcdf0d6 vecs: part 2 2025-07-22 13:19:19 +02:00
nym21 5347523921 vecs: init 2025-07-21 11:02:25 +02:00
nym21 7ef70b953b vec: lazy: remove unneeded phantoms 2025-07-19 17:47:25 +02:00
nym21 ccaca524fe computer: libc sync 2025-07-19 10:10:01 +02:00
nym21 dd51f91cab computer: final fix for external disks crashing 2025-07-18 16:29:53 +02:00
nym21 537d98b41b release: v0.0.81 2025-07-17 23:45:01 +02:00
nym21 9c4cadfc04 vec: fix holes export 2025-07-17 17:29:53 +02:00
nym21 2001370441 mcp: use rust-rmcp instead of brk-rmcp 2025-07-17 16:34:29 +02:00
nym21 cc87b22757 computer: perf improvements 2025-07-17 16:17:21 +02:00
nym21 c0a65b30ad indexer: update example 2025-07-17 11:39:25 +02:00
nym21 c07e66c086 computer: fix stateful 2025-07-17 11:35:40 +02:00
nym21 a0cfc1be2b computer: convert stores to vecs part 2 2025-07-16 16:23:40 +02:00
nym21 1505454793 computer: convert stores to vecs part 1 2025-07-15 22:47:46 +02:00
nym21 e1dff66283 pr: merge #21 from deadmanoz/dockerize
Add Docker support
2025-07-15 15:51:32 +00:00
deadmanoz 5be801a086 Merge branch 'main' into dockerize 2025-07-15 08:50:22 -07:00
deadmanoz 94d4b05c29 Address review feedback 2025-07-15 08:48:39 -07:00
nym21 cebb889f7e cargo: update 2025-07-14 16:00:31 +02:00
nym21 c4ed6ed034 store: remove rotate_memtable as could be the root cause of the issue 2025-07-14 15:48:19 +02:00
deadmanoz 5de9757d46 Remove services from docker 2025-07-04 16:37:39 +08:00
deadmanoz f89276d7b8 Remove redundant services 2025-07-04 15:51:28 +08:00
deadmanoz 30ba034206 Move docker artefacts into /docker directory 2025-07-04 13:00:12 +08:00
deadmanoz fa1e5aaa7f Make Parser::new the only entrypoint 2025-07-04 12:15:32 +08:00
deadmanoz 870c70180f Back to a single image/container setup 2025-07-04 11:40:37 +08:00
deadmanoz d83a833b4d Switch to multiple container setup 2025-06-27 12:56:25 +08:00
deadmanoz ec3a2f29f0 Docker functionality, change location of 'blk_index_to_blk_recap.json' 2025-06-27 12:56:03 +08:00
293 changed files with 10252 additions and 13985 deletions
+11 -6
View File
@@ -1,4 +1,4 @@
# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/
# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
#
# Copyright 2022-2024, axodotdev
# SPDX-License-Identifier: MIT or Apache-2.0
@@ -47,7 +47,7 @@ on:
jobs:
# Run 'dist plan' (or host) to determine what tasks we need to do
plan:
runs-on: "ubuntu-latest"
runs-on: "ubuntu-22.04"
outputs:
val: ${{ steps.plan.outputs.manifest }}
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
@@ -58,12 +58,13 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install dist
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.29.0/cargo-dist-installer.sh | sh"
- name: Cache dist
uses: actions/upload-artifact@v4
with:
@@ -117,6 +118,7 @@ jobs:
git config --global core.longpaths true
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install Rust non-interactively if not already installed
if: ${{ matrix.container }}
@@ -168,13 +170,14 @@ jobs:
needs:
- plan
- build-local-artifacts
runs-on: "ubuntu-latest"
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
@@ -218,12 +221,13 @@ jobs:
if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: "ubuntu-latest"
runs-on: "ubuntu-22.04"
outputs:
val: ${{ steps.host.outputs.manifest }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
@@ -282,10 +286,11 @@ jobs:
# still allowing individual publish jobs to skip themselves (for prereleases).
# "host" however must run to completion, no skipping allowed!
if: ${{ always() && needs.host.result == 'success' }}
runs-on: "ubuntu-latest"
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
+6
View File
@@ -19,7 +19,13 @@ _*
# Logs
.log
# Environment variables/configs
.env
# Profiling
profile.json.gz
flamegraph.svg
*.trace
# AI
CLAUDE.md
Generated
+666 -461
View File
File diff suppressed because it is too large Load Diff
+28 -46
View File
@@ -4,9 +4,11 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.0.80"
package.version = "0.0.88"
package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk"
package.readme = "README.md"
package.rust-version = "1.89"
[profile.release]
lto = "fat"
@@ -21,45 +23,38 @@ debug = true
inherits = "release"
[workspace.dependencies]
arc-swap = "1.7.1"
axum = "0.8.4"
bincode = { version = "2.0.1", features = ["serde"] }
bitcoin = { version = "0.32.6", features = ["serde"] }
bitcoin = { version = "0.32.7", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
brk_bundler = { version = "0.0.80", path = "crates/brk_bundler" }
brk_cli = { version = "0.0.80", path = "crates/brk_cli" }
brk_computer = { version = "0.0.80", path = "crates/brk_computer" }
brk_core = { version = "0.0.80", path = "crates/brk_core" }
brk_exit = { version = "0.0.80", path = "crates/brk_exit" }
brk_fetcher = { version = "0.0.80", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.0.80", path = "crates/brk_indexer" }
brk_interface = { version = "0.0.80", path = "crates/brk_interface" }
brk_logger = { version = "0.0.80", path = "crates/brk_logger" }
brk_mcp = { version = "0.0.80", path = "crates/brk_mcp" }
brk_parser = { version = "0.0.80", path = "crates/brk_parser" }
brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]}
# brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]}
brk_server = { version = "0.0.80", path = "crates/brk_server" }
brk_store = { version = "0.0.80", path = "crates/brk_store" }
brk_vec = { version = "0.0.80", path = "crates/brk_vec" }
brk_bundler = { version = "0.0.88", path = "crates/brk_bundler" }
brk_cli = { version = "0.0.88", path = "crates/brk_cli" }
brk_computer = { version = "0.0.88", path = "crates/brk_computer" }
brk_structs = { version = "0.0.88", path = "crates/brk_structs" }
brk_error = { version = "0.0.88", path = "crates/brk_error" }
brk_fetcher = { version = "0.0.88", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.0.88", path = "crates/brk_indexer" }
brk_interface = { version = "0.0.88", path = "crates/brk_interface" }
brk_logger = { version = "0.0.88", path = "crates/brk_logger" }
brk_mcp = { version = "0.0.88", path = "crates/brk_mcp" }
brk_parser = { version = "0.0.88", path = "crates/brk_parser" }
brk_server = { version = "0.0.88", path = "crates/brk_server" }
brk_store = { version = "0.0.88", path = "crates/brk_store" }
vecdb = { version = "0.1.0", features = ["derive"]}
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
byteview = "=0.6.1"
clap = { version = "4.5.41", features = ["string"] }
clap_derive = "4.5.41"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall = "2.11.1"
fjall = "2.11.2"
jiff = "0.2.15"
log = { version = "0.4.27" }
log = "0.4.27"
minreq = { version = "2.14.0", features = ["https", "serde_json"] }
parking_lot = "0.12.4"
rayon = "1.10.0"
schemars = "1.0.4"
serde = { version = "1.0.219" }
serde = "1.0.219"
serde_bytes = "0.11.17"
serde_derive = "1.0.219"
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
tabled = "0.20.0"
tokio = { version = "1.46.1", features = ["rt-multi-thread"] }
zerocopy = { version = "0.8.26" }
serde_json = { version = "1.0.142", features = ["float_roundtrip"] }
tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
zerocopy = "0.8.26"
zerocopy-derive = "0.8.26"
[workspace.metadata.release]
@@ -69,20 +64,7 @@ pre-release-commit-message = "release: v{{version}}"
tag-message = "release: v{{version}}"
[workspace.metadata.dist]
cargo-dist-version = "0.28.0"
cargo-dist-version = "0.29.0"
ci = "github"
installers = []
targets = [
"aarch64-apple-darwin",
"aarch64-unknown-linux-gnu",
"x86_64-apple-darwin",
"x86_64-unknown-linux-gnu",
]
[workspace.metadata.dist.github-custom-runners]
global = "ubuntu-latest"
aarch64-apple-darwin.runner = "macos-14"
x86_64-unknown-linux-gnu.runner = "ubuntu-latest"
x86_64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
aarch64-unknown-linux-gnu.runner = "ubuntu-latest"
aarch64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
+16 -21
View File
@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network.
@@ -40,7 +34,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible
- **[Website](https://bitcoinresearchkit.org)** \
Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
Also available at: [brekit.org](https://brekit.org) // [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
- **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \
Just like the website, it's entirely free, with no authentication or rate-limiting.
@@ -60,21 +54,22 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
## Crates
- [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates
- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a Bitcoin Research Kit instance
- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer
- [`brk_core`](https://crates.io/crates/brk_core): The Core (Structs and Errors) of the Bitcoin Research Kit
- [`brk_exit`](https://crates.io/crates/brk_exit): An exit blocker built on top of ctrlc
- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher
- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser
- [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit
- [`brk_mcp`](https://crates.io/crates/brk_mcp): A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
- [`brk`](https://crates.io/crates/brk): A wrapper around all other `brk-*` crates
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a BRK instance
- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer built on top of [`brk_indexer`](https://crates.io/crates/brk_indexer)
- [`brk_error`](https://crates.io/crates/brk_error): Errors used throughout BRK
- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher
- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin indexer built on top of [`brk_parser`](https://crates.io/crates/brk_parser)
- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to find and format data from BRK
- [`brk_logger`](https://crates.io/crates/brk_logger): A thin wrapper around [`env_logger`](https://crates.io/crates/env_logger)
- [`brk_mcp`](https://crates.io/crates/brk_mcp): A bridge for LLMs to access BRK
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin block parser and iterator built on top of [`bitcoin-rust`](https://crates.io/crates/bitcoin)
- [`brk_server`](https://crates.io/crates/brk_server): A server with an API for anything from BRK
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_structs`](https://crates.io/crates/brk_structs): Structs used throughout BRK
- [`vecdb`](https://crates.io/crates/vecdb): A KISS index-value storage engine
- [`vecdb_macros`](https://crates.io/crates/vecdb_macros): Macros for [`vecdb`](https://crates.io/crates/vecdb)
## Hosting as a service
+5 -5
View File
@@ -10,8 +10,12 @@
- pull latest version and notify is out of date
- _computer_
- **add rollback of states (in stateful)**
- remove configurable format (raw/compressed) and chose sane ones instead
- linear reads: compressed (height/date/... + txindex_to_height + txindex_to_version + ...)
- random reads: raw (outputindex_to_value + ...)
- add prices paid by percentile (percentile cost basis) back
- add support for per index computation
- fix feerate which is always ZERO due to coinbase transaction
- fix min feerate which is always ZERO due to coinbase transaction
- before computing multiple sources check their length, panic if not equal
- add oracle price dataset (https://utxo.live/oracle/UTXOracle.py)
- add address counts relative to all datasets
@@ -56,10 +60,6 @@
- add extensions support (.json .csv …)
- if format instead of extension then don't download file
- add support for https (rustls)
- _vec_
- add native lock file support (once it's available in stable rust)
- improve compressed mode (slow reads)
- add ema support
- __docs__
- _README_
- add a comparison table with alternatives
+14 -12
View File
@@ -7,52 +7,54 @@ homepage.workspace = true
repository.workspace = true
edition.workspace = true
version.workspace = true
rust-version.workspace = true
build = "build.rs"
[features]
full = [
"bundler",
"core",
"computer",
"exit",
"error",
"fetcher",
"indexer",
"interface",
"logger",
"mcp",
"parser",
"interface",
"server",
"store",
"vec",
"structs",
"vecs",
]
bundler = ["brk_bundler"]
core = ["brk_core"]
computer = ["brk_computer"]
exit = ["brk_exit"]
error = ["brk_error"]
fetcher = ["brk_fetcher"]
indexer = ["brk_indexer"]
interface = ["brk_interface"]
logger = ["brk_logger"]
mcp = ["brk_mcp"]
parser = ["brk_parser"]
interface = ["brk_interface"]
server = ["brk_server"]
store = ["brk_store"]
vec = ["brk_vec"]
structs = ["brk_structs"]
vecs = ["vecdb"]
[dependencies]
brk_bundler = { workspace = true, optional = true }
brk_cli = { workspace = true }
brk_core = { workspace = true, optional = true }
brk_computer = { workspace = true, optional = true }
brk_exit = { workspace = true, optional = true }
brk_error = { workspace = true, optional = true }
brk_fetcher = { workspace = true, optional = true }
brk_indexer = { workspace = true, optional = true }
brk_interface = { workspace = true, optional = true }
brk_logger = { workspace = true, optional = true }
brk_mcp = { workspace = true, optional = true }
brk_parser = { workspace = true, optional = true }
brk_interface = { workspace = true, optional = true }
brk_server = { workspace = true, optional = true }
brk_store = { workspace = true, optional = true }
brk_vec = { workspace = true, optional = true }
brk_structs = { workspace = true, optional = true }
vecdb = { workspace = true, optional = true }
[package.metadata.docs.rs]
all-features = true
+8
View File
@@ -0,0 +1,8 @@
fn main() {
let profile = std::env::var("PROFILE").unwrap_or_default();
if profile == "release" {
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-cpu=native");
}
}
+1 -2
View File
@@ -1,2 +1 @@
cargo build --profile profiling
flamegraph -- ../../target/profiling/brk
sudo cargo flamegraph --profile profiling --root
+6 -6
View File
@@ -7,17 +7,17 @@ pub use brk_bundler as bundler;
#[doc(inline)]
pub use brk_cli as cli;
#[cfg(feature = "core")]
#[cfg(feature = "structs")]
#[doc(inline)]
pub use brk_core as core;
pub use brk_structs as structs;
#[cfg(feature = "computer")]
#[doc(inline)]
pub use brk_computer as computer;
#[cfg(feature = "exit")]
#[cfg(feature = "error")]
#[doc(inline)]
pub use brk_exit as exit;
pub use brk_error as error;
#[cfg(feature = "fetcher")]
#[doc(inline)]
@@ -51,6 +51,6 @@ pub use brk_server as server;
#[doc(inline)]
pub use brk_store as store;
#[cfg(feature = "vec")]
#[cfg(feature = "vecs")]
#[doc(inline)]
pub use brk_vec as vec;
pub use vecdb as vecs;
+4 -2
View File
@@ -6,11 +6,13 @@ edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
rust-version.workspace = true
build = "build.rs"
[dependencies]
log = { workspace = true }
notify = "8.1.0"
brk_rolldown = "0.1.1"
notify = "8.2.0"
brk_rolldown = "0.1.4"
# brk_rolldown = { path = "../../../rolldown/crates/rolldown"}
sugar_path = "1.2.0"
tokio = { workspace = true }
+8
View File
@@ -0,0 +1,8 @@
fn main() {
let profile = std::env::var("PROFILE").unwrap_or_default();
if profile == "release" {
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-cpu=native");
}
}
+14 -12
View File
@@ -1,4 +1,8 @@
use std::{fs, io, path::Path, sync::Arc};
use std::{
fs, io,
path::{Path, PathBuf},
sync::Arc,
};
use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
use log::error;
@@ -8,7 +12,7 @@ use tokio::sync::Mutex;
const VERSION: &str = env!("CARGO_PKG_VERSION");
pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<()> {
pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<PathBuf> {
let source_path = websites_path.join(source_folder);
let dist_path = websites_path.join("dist");
@@ -48,14 +52,12 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i
let write_index = move || {
let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js")) {
if let Some(start) = entry.find("main") {
if let Some(end) = entry.find(".js") {
let main_hashed = &entry[start..end];
contents =
contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
}
}
if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js"))
&& let Some(start) = entry.find("main")
&& let Some(end) = entry.find(".js")
{
let main_hashed = &entry[start..end];
contents = contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
}
let _ = fs::write(&absolute_dist_index_path, contents);
@@ -72,7 +74,7 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i
write_sw();
if !watch {
return Ok(());
return Ok(dist_path);
}
tokio::spawn(async move {
@@ -127,7 +129,7 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i
watcher.start().await;
});
Ok(())
Ok(dist_path)
}
fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
+12 -8
View File
@@ -1,30 +1,34 @@
[package]
name = "brk_cli"
description = "A command line interface to run a Bitcoin Research Kit instance"
description = "A command line interface to run a BRK instance"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
rust-version.workspace = true
build = "build.rs"
[dependencies]
bitcoincore-rpc = { workspace = true }
brk_bundler = { workspace = true }
brk_computer = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_interface = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_server = { workspace = true }
brk_vec = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }
vecdb = { workspace = true }
clap = { version = "4.5.43", features = ["string"] }
clap_derive = "4.5.41"
color-eyre = "0.6.5"
log = { workspace = true }
minreq = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.2"
toml = "0.9.5"
zip = { version = "4.3.0", default-features = false, features = ["deflate"] }
[[bin]]
name = "brk"
+2 -8
View File
@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A command line interface to run a Bitcoin Research Kit instance.
@@ -70,10 +64,10 @@ You can find a pre-built binary for your operating system in the [releases page]
```bash
# Install
cargo install brk # or `cargo install brk_cli`, the result is the same
cargo install brk --locked # or `cargo install brk_cli`, the result is the same
# Update
cargo install brk # or `cargo install-update -a` if you have `cargo-update` installed
cargo install brk --locked # or `cargo install-update -a` if you have `cargo-update` installed
```
### Source
+8
View File
@@ -0,0 +1,8 @@
fn main() {
let profile = std::env::var("PROFILE").unwrap_or_default();
if profile == "release" {
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-cpu=native");
}
}
@@ -1,8 +1,9 @@
use std::{fs, io, path::Path};
use brk_interface::{Index, Interface};
use brk_server::VERSION;
use crate::{VERSION, Website};
use crate::website::Website;
const SCRIPTS: &str = "scripts";
+34 -54
View File
@@ -4,16 +4,15 @@ use std::{
};
use bitcoincore_rpc::{self, Auth, Client};
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
use brk_fetcher::Fetcher;
use brk_server::Website;
use brk_vec::{Computation, Format};
use clap::Parser;
use clap_derive::Parser;
use color_eyre::eyre::eyre;
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize};
use crate::services::Services;
use crate::{default_bitcoin_path, default_brk_path, dot_brk_path, website::Website};
const DOWNLOADS: &str = "downloads";
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
#[command(version, about)]
@@ -33,26 +32,16 @@ pub struct Config {
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
/// Activated services, default: all, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
services: Option<Services>,
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
computation: Option<Computation>,
/// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
format: Option<Format>,
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
/// Activate fetching prices from BRK's API and the computation of all price related datasets, default: true, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short = 'F', long, value_name = "BOOL")]
fetch: Option<bool>,
/// Activate fetching prices from exchanges APIs if `fetch` is also set to `true`, default: true, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "BOOL")]
exchanges: Option<bool>,
/// Website served by the server (if active), default: default, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
@@ -129,20 +118,12 @@ impl Config {
config_saved.brkdir = Some(brkdir);
}
if let Some(services) = config_args.services.take() {
config_saved.services = Some(services);
}
if let Some(computation) = config_args.computation.take() {
config_saved.computation = Some(computation);
}
if let Some(fetch) = config_args.fetch.take() {
config_saved.fetch = Some(fetch);
}
if let Some(format) = config_args.format.take() {
config_saved.format = Some(format);
if let Some(exchanges) = config_args.exchanges.take() {
config_saved.exchanges = Some(exchanges);
}
if let Some(website) = config_args.website.take() {
@@ -224,7 +205,9 @@ impl Config {
if self.rpc_auth().is_err() {
println!(
"No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help."
"Unsuccessful authentication with the RPC client.
First make sure that `bitcoind` is running. If it is then please either set --rpccookiefile or --rpcuser and --rpcpassword as the default values seemed to have failed.
Finally, you can run the program with '-h' for help."
);
std::process::exit(1);
}
@@ -298,22 +281,12 @@ impl Config {
.map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref()))
}
pub fn outputsdir(&self) -> PathBuf {
self.brkdir().join("outputs")
}
pub fn harsdir(&self) -> PathBuf {
self.outputsdir().join("hars")
self.brkdir().join("hars")
}
pub fn process(&self) -> bool {
self.services
.is_none_or(|m| m == Services::All || m == Services::Processor)
}
pub fn serve(&self) -> bool {
self.services
.is_none_or(|m| m == Services::All || m == Services::Server)
pub fn downloads_dir(&self) -> PathBuf {
dot_brk_path().join(DOWNLOADS)
}
fn path_cookiefile(&self) -> PathBuf {
@@ -349,17 +322,13 @@ impl Config {
self.fetch.is_none_or(|b| b)
}
pub fn exchanges(&self) -> bool {
self.exchanges.is_none_or(|b| b)
}
pub fn fetcher(&self) -> Option<Fetcher> {
self.fetch()
.then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap())
}
pub fn computation(&self) -> Computation {
self.computation.unwrap_or_default()
}
pub fn format(&self) -> Format {
self.format.unwrap_or_default()
.then(|| Fetcher::import(self.exchanges(), Some(self.harsdir().as_path())).unwrap())
}
pub fn check_collisions(&self) -> bool {
@@ -374,3 +343,14 @@ impl Config {
self.watch.is_some_and(|b| b)
}
}
fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: Deserialize<'de> + Default,
{
match T::deserialize(deserializer) {
Ok(v) => Ok(v),
Err(_) => Ok(T::default()),
}
}
+136 -5
View File
@@ -1,12 +1,28 @@
use std::{fs, thread};
#![doc = include_str!("../README.md")]
use brk_core::{dot_brk_log_path, dot_brk_path};
use std::{
fs,
io::Cursor,
path::Path,
thread::{self, sleep},
time::Duration,
};
use bitcoincore_rpc::{self, RpcApi};
use brk_bundler::bundle;
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_interface::Interface;
use brk_server::{Server, VERSION};
use log::info;
use vecdb::Exit;
mod bridge;
mod config;
mod run;
mod services;
mod paths;
mod website;
use run::*;
use crate::{bridge::Bridge, config::Config, paths::*};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -21,3 +37,118 @@ pub fn main() -> color_eyre::Result<()> {
.join()
.unwrap()
}
pub fn run() -> color_eyre::Result<()> {
let config = Config::import()?;
let rpc = config.rpc()?;
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc);
let mut indexer = Indexer::forced_import(&config.brkdir())?;
let wait_for_synced_node = |rpc_client: &bitcoincore_rpc::Client| -> color_eyre::Result<()> {
let is_synced = || -> color_eyre::Result<bool> {
let info = rpc_client.get_blockchain_info()?;
Ok(info.headers == info.blocks)
};
if !is_synced()? {
info!("Waiting for node to be synced...");
while !is_synced()? {
sleep(Duration::from_secs(1))
}
}
Ok(())
};
let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?;
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
.block_on(async {
let interface = Interface::build(&indexer, &computer);
let website = config.website();
let downloads_path = config.downloads_dir();
let bundle_path = if website.is_some() {
let websites_dev_path = Path::new("../../websites");
let websites_path = if fs::exists(websites_dev_path)? {
websites_dev_path.to_path_buf()
} else {
let downloaded_websites_path =
downloads_path.join(format!("brk-{VERSION}")).join("websites");
if !fs::exists(&downloaded_websites_path)? {
info!("Downloading websites from Github...");
let url = format!(
"https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{VERSION}.zip",
);
let response = minreq::get(url).send()?;
let bytes = response.as_bytes();
let cursor = Cursor::new(bytes);
let mut zip = zip::ZipArchive::new(cursor).unwrap();
zip.extract(downloads_path).unwrap();
}
downloaded_websites_path
};
interface.generate_bridge_file(website, websites_path.as_path())?;
let watch = config.watch();
Some(bundle(&websites_path, website.to_folder_name(), watch).await?)
} else {
None
};
let server = Server::new(
interface,
bundle_path,
);
let mcp = config.mcp();
tokio::spawn(async move {
server.serve(mcp).await.unwrap();
});
sleep(Duration::from_secs(1));
loop {
wait_for_synced_node(rpc)?;
let block_count = rpc.get_block_count()?;
info!("{} blocks found.", block_count + 1);
let starting_indexes =
indexer.index(&parser, rpc, &exit, config.check_collisions()).unwrap();
computer.compute(&indexer, starting_indexes, &exit).unwrap();
if let Some(delay) = config.delay() {
sleep(Duration::from_secs(delay))
}
info!("Waiting for new blocks...");
while block_count == rpc.get_block_count()? {
sleep(Duration::from_secs(1))
}
}
})
}
-103
View File
@@ -1,103 +0,0 @@
use std::{thread::sleep, time::Duration};
use bitcoincore_rpc::{self, RpcApi};
use brk_computer::Computer;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_server::Server;
use log::info;
use crate::config::Config;
pub fn run() -> color_eyre::Result<()> {
let config = Config::import()?;
let rpc = config.rpc()?;
let exit = Exit::new();
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
let format = config.format();
let mut indexer = Indexer::forced_import(&config.outputsdir())?;
let wait_for_synced_node = || -> color_eyre::Result<()> {
let is_synced = || -> color_eyre::Result<bool> {
let info = rpc.get_blockchain_info()?;
Ok(info.headers == info.blocks)
};
if !is_synced()? {
info!("Waiting for node to be synced...");
while !is_synced()? {
sleep(Duration::from_secs(1))
}
}
Ok(())
};
let mut computer = Computer::forced_import(
&config.outputsdir(),
&indexer,
config.computation(),
config.fetcher(),
format,
)?;
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
.block_on(async {
let server = if config.serve() {
let served_indexer = indexer.clone();
let served_computer = computer.clone();
let server = Server::new(served_indexer, served_computer, config.website())?;
let watch = config.watch();
let mcp = config.mcp();
let opt = Some(tokio::spawn(async move {
server.serve(watch, mcp).await.unwrap();
}));
sleep(Duration::from_secs(1));
opt
} else {
None
};
if config.process() {
loop {
wait_for_synced_node()?;
let block_count = rpc.get_block_count()?;
info!("{} blocks found.", block_count + 1);
let starting_indexes =
indexer.index(&parser, rpc, &exit, config.check_collisions())?;
computer.compute(&mut indexer, starting_indexes, &exit)?;
if let Some(delay) = config.delay() {
sleep(Duration::from_secs(delay))
}
info!("Waiting for new blocks...");
while block_count == rpc.get_block_count()? {
sleep(Duration::from_secs(1))
}
}
}
if let Some(handle) = server {
handle.await.unwrap();
}
Ok(())
})
}
-23
View File
@@ -1,23 +0,0 @@
use clap_derive::{Parser, ValueEnum};
use serde::{Deserialize, Serialize};
#[derive(
Default,
Debug,
Clone,
Copy,
Parser,
ValueEnum,
Serialize,
Deserialize,
PartialEq,
Eq,
PartialOrd,
Ord,
)]
pub enum Services {
#[default]
All,
Processor,
Server,
}
@@ -17,7 +17,7 @@ impl Website {
!self.is_none()
}
pub fn to_folder_name(&self) -> &str {
pub fn to_folder_name(self) -> &'static str {
match self {
Self::Custom => "custom",
Self::Default => "default",
+6 -9
View File
@@ -1,28 +1,25 @@
[package]
name = "brk_computer"
description = "A Bitcoin dataset computer, built on top of brk_indexer"
description = "A Bitcoin dataset computer built on top of brk_indexer"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
rust-version.workspace = true
build = "build.rs"
[dependencies]
bincode = { workspace = true }
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_structs = { workspace = true }
brk_error = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vec = { workspace = true }
color-eyre = { workspace = true }
vecdb = { workspace = true }
derive_deref = { workspace = true }
either = "1.15.0"
fjall = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
-6
View File
@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A dataset computer, built on top of `brk_indexer` and `brk_fetcher`. It computes any dataset you can think of and if it doesn't feel free to create an issue.
+8
View File
@@ -0,0 +1,8 @@
fn main() {
let profile = std::env::var("PROFILE").unwrap_or_default();
if profile == "release" {
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-cpu=native");
}
}
+55
View File
@@ -0,0 +1,55 @@
use std::{
path::Path,
thread::{self, sleep},
time::{Duration, Instant},
};
use brk_computer::Computer;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use vecdb::Exit;
pub fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
exit.set_ctrlc_handler();
// Can't increase main thread's stack size, thus we need to use another thread
thread::Builder::new()
.stack_size(256 * 1024 * 1024)
.spawn(move || -> Result<()> {
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
// let outputs_dir = Path::new("../../_outputs");
let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);
let mut indexer = Indexer::forced_import(&outputs_dir)?;
let fetcher = Fetcher::import(true, None)?;
let mut computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
loop {
let i = Instant::now();
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
computer.compute(&indexer, starting_indexes, &exit)?;
dbg!(i.elapsed());
sleep(Duration::from_secs(10));
}
})?
.join()
.unwrap()
}
-57
View File
@@ -1,57 +0,0 @@
use std::{path::Path, thread};
use brk_computer::Computer;
use brk_core::{default_bitcoin_path, default_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vec::{Computation, Format};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
brk_logger::init(Some(Path::new(".log")));
// let bitcoin_dir = default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
// Can't increase main thread's stack programatically, thus we need to use another thread
thread::Builder::new()
.stack_size(256 * 1024 * 1024)
.spawn(move || -> color_eyre::Result<()> {
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs");
let outputs_dir = _outputs_dir.as_path();
// let outputs_dir = Path::new("../../_outputs");
let format = Format::Raw;
let mut indexer = Indexer::forced_import(outputs_dir)?;
let fetcher = Fetcher::import(None)?;
let mut computer = Computer::forced_import(
outputs_dir,
&indexer,
Computation::Lazy,
Some(fetcher),
format,
)?;
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
computer.compute(&mut indexer, starting_indexes, &exit)?;
Ok(())
})?
.join()
.unwrap()
}
@@ -1,14 +1,16 @@
use std::path::Path;
use brk_core::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
Timestamp, Version, Weight,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
use brk_structs::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version,
Weight,
};
use vecdb::{
AnyCollectableVec, Computation, Database, EagerVec, Exit, Format, PAGE_SIZE, VecIterator,
};
use crate::vecs::grouped::Source;
use crate::grouped::Source;
use super::{
Indexes,
@@ -20,6 +22,8 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub height_to_interval: EagerVec<Height, Timestamp>,
pub height_to_vbytes: EagerVec<Height, StoredU64>,
pub difficultyepoch_to_timestamp: EagerVec<DifficultyEpoch, Timestamp>,
@@ -27,28 +31,31 @@ pub struct Vecs {
pub timeindexes_to_timestamp: ComputedVecsFromDateIndex<Timestamp>,
pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
pub indexes_to_block_size: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_block_size: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_block_vbytes: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_block_weight: ComputedVecsFromHeight<Weight>,
}
impl Vecs {
pub fn forced_import(
path: &Path,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let db = Database::open(&parent.join("blocks"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
height_to_interval: EagerVec::forced_import(
path,
&db,
"interval",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
path,
&db,
"timestamp",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -58,7 +65,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
path,
&db,
"block_interval",
Source::None,
version + VERSION + Version::ZERO,
@@ -71,7 +78,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_block_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"block_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -81,7 +88,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
path,
&db,
"block_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -91,7 +98,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
path,
&db,
"block_size",
Source::None,
version + VERSION + Version::ZERO,
@@ -101,13 +108,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
path,
&db,
"vbytes",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
path,
&db,
"block_vbytes",
Source::None,
version + VERSION + Version::ZERO,
@@ -117,17 +124,19 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
path,
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_timestamp: EagerVec::forced_import(
path,
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
db,
})
}
@@ -137,7 +146,19 @@ impl Vecs {
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.timeindexes_to_timestamp.compute_all(
indexer,
indexes,
@@ -149,7 +170,8 @@ impl Vecs {
&indexes.dateindex_to_date,
|(di, d, ..)| (di, Timestamp::from(d)),
exit,
)
)?;
Ok(())
},
)?;
@@ -164,7 +186,8 @@ impl Vecs {
&indexer.vecs.height_to_weight,
|h| (h, StoredU32::from(1_u32)),
exit,
)
)?;
Ok(())
},
)?;
@@ -1,26 +1,25 @@
use std::path::Path;
use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
use crate::vecs::{
fetched,
grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, Source},
stateful, transactions,
};
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator};
use super::{
Indexes,
grouped::{ComputedVecsFromHeight, VecBuilderOptions},
indexes,
grouped::{
ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
Source, VecBuilderOptions,
},
indexes, price, stateful, transactions,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub indexes_to_coinblocks_created: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_coinblocks_stored: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_liveliness: ComputedVecsFromHeight<StoredF64>,
@@ -49,18 +48,21 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
price: Option<&price::Vecs>,
) -> Result<Self> {
let db = Database::open(&parent.join("cointime"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let compute_dollars = price.is_some();
Ok(Self {
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
path,
&db,
"coinblocks_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -70,7 +72,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
path,
&db,
"coinblocks_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -80,7 +82,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
path,
&db,
"liveliness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -90,7 +92,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
path,
&db,
"vaultedness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -100,7 +102,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
path,
&db,
"activity_to_vaultedness_ratio",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -110,7 +112,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
path,
&db,
"vaulted_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -121,7 +123,7 @@ impl Vecs {
indexes,
)?,
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
path,
&db,
"active_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -132,7 +134,7 @@ impl Vecs {
indexes,
)?,
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
path,
&db,
"thermo_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -142,7 +144,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
path,
&db,
"investor_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -152,7 +154,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
path,
&db,
"vaulted_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -162,7 +164,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
path,
&db,
"active_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -172,7 +174,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
path,
&db,
"vaulted_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -182,7 +184,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
&db,
"vaulted_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -192,7 +194,7 @@ impl Vecs {
true,
)?,
indexes_to_active_price: ComputedVecsFromHeight::forced_import(
path,
&db,
"active_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -202,7 +204,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
&db,
"active_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -212,7 +214,7 @@ impl Vecs {
true,
)?,
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
path,
&db,
"true_market_mean",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -222,7 +224,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
&db,
"true_market_mean",
Source::None,
version + VERSION + Version::ZERO,
@@ -232,7 +234,7 @@ impl Vecs {
true,
)?,
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
path,
&db,
"cointime_value_destroyed",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -242,7 +244,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
path,
&db,
"cointime_value_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -252,7 +254,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
path,
&db,
"cointime_value_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -262,7 +264,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
path,
&db,
"cointime_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -272,7 +274,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
path,
&db,
"cointime_cap",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -282,7 +284,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
&db,
"cointime_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -291,6 +293,8 @@ impl Vecs {
indexes,
true,
)?,
db,
})
}
@@ -300,12 +304,36 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
transactions: &transactions::Vecs,
stateful: &stateful::Vecs,
exit: &Exit,
) -> color_eyre::Result<()> {
let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply;
) -> Result<()> {
self.compute_(
indexer,
indexes,
starting_indexes,
price,
transactions,
stateful,
exit,
)?;
self.db.flush_then_punch()?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
price: Option<&price::Vecs>,
transactions: &transactions::Vecs,
stateful: &stateful::Vecs,
exit: &Exit,
) -> Result<()> {
let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply;
self.indexes_to_coinblocks_created.compute_all(
indexer,
@@ -318,12 +346,13 @@ impl Vecs {
circulating_supply,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)
)?;
Ok(())
},
)?;
let indexes_to_coinblocks_destroyed =
&stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed;
&stateful.utxo_cohorts.all.1.indexes_to_coinblocks_destroyed;
self.indexes_to_coinblocks_stored.compute_all(
indexer,
@@ -344,7 +373,8 @@ impl Vecs {
(i, created.checked_sub(destroyed).unwrap())
},
exit,
)
)?;
Ok(())
},
)?;
@@ -363,7 +393,8 @@ impl Vecs {
.height_extra
.unwrap_cumulative(),
exit,
)
)?;
Ok(())
},
)?;
let liveliness = &self.indexes_to_liveliness;
@@ -379,7 +410,8 @@ impl Vecs {
liveliness.height.as_ref().unwrap(),
|(i, v, ..)| (i, StoredF64::from(1.0).checked_sub(v).unwrap()),
exit,
)
)?;
Ok(())
},
)?;
let vaultedness = &self.indexes_to_vaultedness;
@@ -395,14 +427,15 @@ impl Vecs {
liveliness.height.as_ref().unwrap(),
vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_vaulted_supply.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -411,14 +444,15 @@ impl Vecs {
circulating_supply,
vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_active_supply.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -427,13 +461,14 @@ impl Vecs {
circulating_supply,
liveliness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
if let Some(fetched) = fetched {
if let Some(price) = price {
let realized_cap = stateful
.utxo_vecs
.utxo_cohorts
.all
.1
.height_to_realized_cap
@@ -441,7 +476,7 @@ impl Vecs {
.unwrap();
let realized_price = stateful
.utxo_vecs
.utxo_cohorts
.all
.1
.indexes_to_realized_price
@@ -468,7 +503,8 @@ impl Vecs {
.unwrap_cumulative(),
|(i, v, ..)| (i, v),
exit,
)
)?;
Ok(())
},
)?;
@@ -483,7 +519,8 @@ impl Vecs {
realized_cap,
self.indexes_to_thermo_cap.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -498,7 +535,8 @@ impl Vecs {
realized_cap,
self.indexes_to_vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -513,7 +551,8 @@ impl Vecs {
realized_cap,
self.indexes_to_liveliness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -528,14 +567,15 @@ impl Vecs {
realized_price,
self.indexes_to_vaultedness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_vaulted_price_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_vaulted_price.dateindex.unwrap_last()),
@@ -552,14 +592,15 @@ impl Vecs {
realized_price,
self.indexes_to_liveliness.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_active_price_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_active_price.dateindex.unwrap_last()),
@@ -580,14 +621,15 @@ impl Vecs {
.as_ref()
.unwrap(),
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_true_market_mean_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_true_market_mean.dateindex.unwrap_last()),
@@ -603,10 +645,11 @@ impl Vecs {
// The price taken won't be correct for time based indexes
vec.compute_multiply(
starting_indexes.height,
&fetched.chainindexes_to_close.height,
&price.chainindexes_to_close.height,
indexes_to_coinblocks_destroyed.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -618,10 +661,11 @@ impl Vecs {
|vec, _, _, starting_indexes, exit| {
vec.compute_multiply(
starting_indexes.height,
&fetched.chainindexes_to_close.height,
&price.chainindexes_to_close.height,
self.indexes_to_coinblocks_created.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -633,10 +677,11 @@ impl Vecs {
|vec, _, _, starting_indexes, exit| {
vec.compute_multiply(
starting_indexes.height,
&fetched.chainindexes_to_close.height,
&price.chainindexes_to_close.height,
self.indexes_to_coinblocks_stored.height.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -655,7 +700,8 @@ impl Vecs {
.height_extra
.unwrap_cumulative(),
exit,
)
)?;
Ok(())
},
)?;
@@ -670,14 +716,15 @@ impl Vecs {
self.indexes_to_cointime_price.height.as_ref().unwrap(),
circulating_supply,
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_cointime_price_ratio.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
Some(self.indexes_to_cointime_price.dateindex.unwrap_last()),
@@ -1,11 +1,11 @@
use std::path::Path;
use brk_core::{StoredU8, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
use brk_structs::{StoredU16, Version};
use vecdb::{AnyCollectableVec, AnyVec, Computation, Database, Exit, Format};
use crate::vecs::grouped::Source;
use crate::grouped::Source;
use super::{
Indexes,
@@ -17,23 +17,27 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
pub constant_0: ComputedVecsFromHeight<StoredU8>,
pub constant_1: ComputedVecsFromHeight<StoredU8>,
pub constant_50: ComputedVecsFromHeight<StoredU8>,
pub constant_100: ComputedVecsFromHeight<StoredU8>,
db: Database,
pub constant_0: ComputedVecsFromHeight<StoredU16>,
pub constant_1: ComputedVecsFromHeight<StoredU16>,
pub constant_50: ComputedVecsFromHeight<StoredU16>,
pub constant_100: ComputedVecsFromHeight<StoredU16>,
}
impl Vecs {
pub fn forced_import(
path: &Path,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let db = Database::open(&parent.join("constants"))?;
Ok(Self {
constant_0: ComputedVecsFromHeight::forced_import(
path,
&db,
"constant_0",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -43,7 +47,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_1: ComputedVecsFromHeight::forced_import(
path,
&db,
"constant_1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -53,7 +57,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_50: ComputedVecsFromHeight::forced_import(
path,
&db,
"constant_50",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -63,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_100: ComputedVecsFromHeight::forced_import(
path,
&db,
"constant_100",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -72,6 +76,8 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_last(),
)?,
db,
})
}
@@ -81,7 +87,19 @@ impl Vecs {
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.constant_0.compute_all(
indexer,
indexes,
@@ -92,9 +110,10 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(0)),
|i| (i, StoredU16::new(0)),
exit,
)
)?;
Ok(())
},
)?;
@@ -108,9 +127,10 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(1)),
|i| (i, StoredU16::new(1)),
exit,
)
)?;
Ok(())
},
)?;
@@ -124,9 +144,10 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(50)),
|i| (i, StoredU16::new(50)),
exit,
)
)?;
Ok(())
},
)?;
@@ -140,9 +161,10 @@ impl Vecs {
starting_indexes.height,
indexes.height_to_date.len(),
indexes.height_to_date.version(),
|i| (i, StoredU8::new(100)),
|i| (i, StoredU16::new(100)),
exit,
)
)?;
Ok(())
},
)?;
+139
View File
@@ -0,0 +1,139 @@
use std::path::Path;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_structs::{DateIndex, Height, OHLCCents, Version};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec,
RawVec, StoredIndex, VecIterator,
};
use super::{Indexes, indexes};
#[derive(Clone)]
pub struct Vecs {
db: Database,
fetcher: Fetcher,
pub dateindex_to_ohlc_in_cents: RawVec<DateIndex, OHLCCents>,
pub height_to_ohlc_in_cents: RawVec<Height, OHLCCents>,
}
impl Vecs {
pub fn forced_import(parent: &Path, fetcher: Fetcher, version: Version) -> Result<Self> {
let db = Database::open(&parent.join("fetched"))?;
Ok(Self {
fetcher,
dateindex_to_ohlc_in_cents: RawVec::forced_import(
&db,
"ohlc_in_cents",
version + Version::ZERO,
)?,
height_to_ohlc_in_cents: RawVec::forced_import(
&db,
"ohlc_in_cents",
version + Version::ZERO,
)?,
db,
})
}
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let height_to_timestamp = &indexer.vecs.height_to_timestamp;
let index = starting_indexes
.height
.min(Height::from(self.height_to_ohlc_in_cents.len()));
height_to_timestamp
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let v = v.into_owned();
self.height_to_ohlc_in_cents.forced_push_at(
i,
self.fetcher
.get_height(
i,
v,
i.decremented().map(|prev_i| {
height_to_timestamp.into_iter().unwrap_get_inner(prev_i)
}),
)
.unwrap(),
exit,
)?;
Ok(())
})?;
self.height_to_ohlc_in_cents.safe_flush(exit)?;
let index = starting_indexes
.dateindex
.min(DateIndex::from(self.dateindex_to_ohlc_in_cents.len()));
let mut prev = None;
indexes
.dateindex_to_date
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let d = v.into_owned();
if prev.is_none() {
let i = i.unwrap_to_usize();
prev.replace(if i > 0 {
self.dateindex_to_ohlc_in_cents
.into_iter()
.unwrap_get_inner_(i - 1)
} else {
OHLCCents::default()
});
}
let ohlc = if i.unwrap_to_usize() + 100 >= self.dateindex_to_ohlc_in_cents.len()
&& let Ok(mut ohlc) = self.fetcher.get_date(d)
{
let prev_open = *prev.as_ref().unwrap().close;
*ohlc.open = prev_open;
*ohlc.high = (*ohlc.high).max(prev_open);
*ohlc.low = (*ohlc.low).min(prev_open);
ohlc
} else {
prev.clone().unwrap()
};
prev.replace(ohlc.clone());
self.dateindex_to_ohlc_in_cents
.forced_push_at(i, ohlc, exit)?;
Ok(())
})?;
self.dateindex_to_ohlc_in_cents.safe_flush(exit)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
vec![
&self.dateindex_to_ohlc_in_cents as &dyn AnyCollectableVec,
&self.height_to_ohlc_in_cents,
]
}
}
@@ -1,13 +1,12 @@
use std::path::Path;
use brk_error::Result;
use brk_core::{FromCoarserIndex, Result, Version};
use brk_exit::Exit;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, Format, StoredIndex,
use brk_structs::Version;
use vecdb::{
AnyBoxedIterableVec, AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, Database, Exit, Format, FromCoarserIndex, StoredIndex,
};
use crate::vecs::grouped::{EagerVecBuilder, VecBuilderOptions};
use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
use super::ComputedType;
@@ -39,16 +38,16 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
version: Version,
format: Format,
computation: Computation,
source: Option<BoxedAnyIterableVec<S1I, T>>,
source: Option<AnyBoxedIterableVec<S1I, T>>,
source_extra: &EagerVecBuilder<S1I, T>,
len_source: BoxedAnyIterableVec<I, S2T>,
len_source: AnyBoxedIterableVec<I, S2T>,
options: ComputedVecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let only_one_active = options.is_only_one_active();
let suffix = |s: &str| format!("{name}_{s}");
@@ -66,7 +65,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
db,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -91,7 +90,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -100,7 +99,7 @@ where
source
.as_ref()
.unwrap_or_else(|| {
dbg!(path, name, I::to_string());
dbg!(db, name, I::to_string());
panic!()
})
.clone()
@@ -124,7 +123,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
db,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -149,7 +148,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
db,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -174,7 +173,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
db,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -206,7 +205,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
db,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -241,7 +240,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
db,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,
@@ -1,9 +1,9 @@
use std::path::Path;
use brk_core::{CheckedSub, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use brk_error::{Error, Result};
use brk_structs::{CheckedSub, StoredU64, Version};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format,
GenericStoredVec, StoredIndex, StoredRaw,
};
use crate::utils::get_percentile;
@@ -37,12 +37,12 @@ where
T: ComputedType,
{
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
version: Version,
format: Format,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let only_one_active = options.is_only_one_active();
let suffix = |s: &str| format!("{name}_{s}");
@@ -59,7 +59,7 @@ where
first: options.first.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -69,13 +69,13 @@ where
}),
last: options.last.then(|| {
Box::new(
EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(),
EagerVec::forced_import(db, name, version + Version::ZERO, format).unwrap(),
)
}),
min: options.min.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -86,7 +86,7 @@ where
max: options.max.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -97,7 +97,7 @@ where
median: options.median.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("median"),
version + VERSION + Version::ZERO,
format,
@@ -108,7 +108,7 @@ where
average: options.average.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -119,7 +119,7 @@ where
sum: options.sum.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -134,7 +134,7 @@ where
cumulative: options.cumulative.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,
@@ -145,7 +145,7 @@ where
_90p: options._90p.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("90p"),
version + VERSION + Version::ZERO,
format,
@@ -156,7 +156,7 @@ where
_75p: options._75p.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("75p"),
version + VERSION + Version::ZERO,
format,
@@ -167,7 +167,7 @@ where
_25p: options._25p.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("25p"),
version + VERSION + Version::ZERO,
format,
@@ -178,7 +178,7 @@ where
_10p: options._10p.then(|| {
Box::new(
EagerVec::forced_import(
path,
db,
&maybe_suffix("10p"),
version + VERSION + Version::ZERO,
format,
@@ -201,7 +201,7 @@ where
return Ok(());
};
self.validate_computed_version_or_reset_file(source.version())?;
self.validate_computed_version_or_reset(source.version())?;
let index = self.starting_index(max_from);
@@ -211,8 +211,9 @@ where
cumulative_vec.iter().unwrap_get_inner(index)
});
source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
cumulative = cumulative.clone() + v.into_owned();
cumulative_vec.forced_push_at(i, cumulative.clone(), exit)
cumulative += v.into_owned();
cumulative_vec.forced_push_at(i, cumulative, exit)?;
Ok(())
})?;
self.safe_flush(exit)?;
@@ -225,13 +226,13 @@ where
max_from: I,
source: &impl AnyIterableVec<I2, T>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
count_indexes: &impl AnyIterableVec<I, StoredU64>,
exit: &Exit,
) -> Result<()>
where
I2: StoredIndex + StoredType + CheckedSub<I2>,
I2: StoredIndex + StoredRaw + CheckedSub<I2>,
{
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
source.version() + first_indexes.version() + count_indexes.version(),
)?;
@@ -250,10 +251,10 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index)| -> Result<()> {
.try_for_each(|(index, first_index)| -> Result<()> {
let first_index = first_index.into_owned();
let count_index = count_indexes_iter.unwrap_get_inner(i);
let count_index = count_indexes_iter.unwrap_get_inner(index);
if let Some(first) = self.first.as_mut() {
let f = source_iter
@@ -263,7 +264,7 @@ where
}
if let Some(last) = self.last.as_mut() {
let count_index = *count_index;
let count_index = *count_index as usize;
if count_index == 0 {
panic!("should compute last if count can be 0")
}
@@ -293,7 +294,7 @@ where
if needs_values {
source_iter.set(first_index);
let mut values = (&mut source_iter)
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
@@ -302,14 +303,14 @@ where
if let Some(max) = self.max.as_mut() {
max.forced_push_at(
i,
values
index,
*values
.last()
.context("expect some")
.ok_or(Error::Str("expect some"))
.inspect_err(|_| {
dbg!(
&values,
max.path(),
max.name(),
first_indexes.name(),
first_index,
count_indexes.name(),
@@ -318,34 +319,33 @@ where
source.name()
);
})
.unwrap()
.clone(),
.unwrap(),
exit,
)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.forced_push_at(i, get_percentile(&values, 0.90), exit)?;
_90p.forced_push_at(index, get_percentile(&values, 0.90), exit)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.forced_push_at(i, get_percentile(&values, 0.75), exit)?;
_75p.forced_push_at(index, get_percentile(&values, 0.75), exit)?;
}
if let Some(median) = self.median.as_mut() {
median.forced_push_at(i, get_percentile(&values, 0.50), exit)?;
median.forced_push_at(index, get_percentile(&values, 0.50), exit)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.forced_push_at(i, get_percentile(&values, 0.25), exit)?;
_25p.forced_push_at(index, get_percentile(&values, 0.25), exit)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.forced_push_at(i, get_percentile(&values, 0.10), exit)?;
_10p.forced_push_at(index, get_percentile(&values, 0.10), exit)?;
}
if let Some(min) = self.min.as_mut() {
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
min.forced_push_at(index, *values.first().unwrap(), exit)?;
}
}
@@ -354,19 +354,19 @@ where
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(average) = self.average.as_mut() {
let avg = sum.clone() / len;
average.forced_push_at(i, avg, exit)?;
let avg = sum / len;
average.forced_push_at(index, avg, exit)?;
}
if needs_sum_or_cumulative {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
sum_vec.forced_push_at(index, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
cumulative_vec.forced_push_at(i, t, exit)?;
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(index, t, exit)?;
}
}
}
@@ -386,11 +386,11 @@ where
max_from: I,
source: &EagerVecBuilder<I2, T>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
count_indexes: &impl AnyIterableVec<I, StoredU64>,
exit: &Exit,
) -> Result<()>
where
I2: StoredIndex + StoredType + CheckedSub<I2>,
I2: StoredIndex + StoredRaw + CheckedSub<I2>,
{
if self._90p.is_some()
|| self._75p.is_some()
@@ -401,7 +401,7 @@ where
panic!("unsupported");
}
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
VERSION + first_indexes.version() + count_indexes.version(),
)?;
@@ -424,10 +424,10 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index, ..)| -> Result<()> {
.try_for_each(|(index, first_index, ..)| -> Result<()> {
let first_index = first_index.into_owned();
let count_index = count_indexes_iter.unwrap_get_inner(i);
let count_index = count_indexes_iter.unwrap_get_inner(index);
if let Some(first) = self.first.as_mut() {
let v = source_first_iter
@@ -438,7 +438,7 @@ where
}
if let Some(last) = self.last.as_mut() {
let count_index = *count_index;
let count_index = *count_index as usize;
if count_index == 0 {
panic!("should compute last if count can be 0")
}
@@ -462,22 +462,22 @@ where
let source_max_iter = source_max_iter.as_mut().unwrap();
source_max_iter.set(first_index);
let mut values = source_max_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
max.forced_push_at(index, *values.last().unwrap(), exit)?;
}
if let Some(min) = self.min.as_mut() {
let source_min_iter = source_min_iter.as_mut().unwrap();
source_min_iter.set(first_index);
let mut values = source_min_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
min.forced_push_at(index, *values.first().unwrap(), exit)?;
}
}
@@ -486,7 +486,7 @@ where
let source_average_iter = source_average_iter.as_mut().unwrap();
source_average_iter.set(first_index);
let values = source_average_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
@@ -495,27 +495,27 @@ where
// TODO: Multiply by count then divide by cumulative
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
let avg = cumulative / len;
average.forced_push_at(i, avg, exit)?;
average.forced_push_at(index, avg, exit)?;
}
if needs_sum_or_cumulative {
let source_sum_iter = source_sum_iter.as_mut().unwrap();
source_sum_iter.set(first_index);
let values = source_sum_iter
.take(*count_index)
.take(*count_index as usize)
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
sum_vec.forced_push_at(index, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
cumulative_vec.forced_push_at(i, t, exit)?;
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(index, t, exit)?;
}
}
}
@@ -663,42 +663,42 @@ where
Ok(())
}
pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
pub fn validate_computed_version_or_reset(&mut self, version: Version) -> Result<()> {
if let Some(first) = self.first.as_mut() {
first.validate_computed_version_or_reset_file(Version::ZERO + version)?;
first.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(last) = self.last.as_mut() {
last.validate_computed_version_or_reset_file(Version::ZERO + version)?;
last.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(min) = self.min.as_mut() {
min.validate_computed_version_or_reset_file(Version::ZERO + version)?;
min.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(max) = self.max.as_mut() {
max.validate_computed_version_or_reset_file(Version::ZERO + version)?;
max.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(median) = self.median.as_mut() {
median.validate_computed_version_or_reset_file(Version::ZERO + version)?;
median.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(average) = self.average.as_mut() {
average.validate_computed_version_or_reset_file(Version::ZERO + version)?;
average.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(sum) = self.sum.as_mut() {
sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
sum.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(cumulative) = self.cumulative.as_mut() {
cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
cumulative.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_90p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_75p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_25p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_10p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
Ok(())
@@ -0,0 +1,23 @@
use std::ops::{Add, AddAssign, Div};
use vecdb::StoredCompressed;
pub trait ComputedType
where
Self: StoredCompressed
+ From<usize>
+ Div<usize, Output = Self>
+ Add<Output = Self>
+ AddAssign
+ Ord,
{
}
impl<T> ComputedType for T where
T: StoredCompressed
+ From<usize>
+ Div<usize, Output = Self>
+ Add<Output = Self>
+ AddAssign
+ Ord
{
}
@@ -1,16 +1,15 @@
use std::path::Path;
use brk_error::Result;
use brk_core::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex,
YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
use brk_structs::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec,
Exit, Format,
};
use crate::vecs::{Indexes, grouped::ComputedVecBuilder, indexes};
use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions};
@@ -37,7 +36,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
source: Source<DateIndex, T>,
version: Version,
@@ -45,13 +44,13 @@ where
computation: Computation,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let dateindex = source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
});
let dateindex_extra = EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -64,7 +63,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -75,7 +74,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -86,7 +85,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -97,7 +96,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -108,7 +107,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -119,7 +118,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -141,7 +140,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<DateIndex, T>,
@@ -169,7 +168,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(dateindex) = dateindex {
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;
@@ -1,16 +1,16 @@
use std::path::Path;
use brk_error::Result;
use brk_core::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result,
SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
use brk_structs::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec,
Exit, Format,
};
use crate::vecs::{
use crate::{
Indexes,
grouped::{ComputedVecBuilder, Source},
indexes,
@@ -45,7 +45,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
source: Source<Height, T>,
version: Version,
@@ -53,13 +53,13 @@ where
computation: Computation,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let height = source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
});
let height_extra = EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -67,7 +67,7 @@ where
)?;
let dateindex = EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -78,7 +78,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -89,7 +89,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -100,7 +100,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -111,7 +111,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -122,7 +122,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -133,7 +133,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -143,12 +143,12 @@ where
indexes.decadeindex_to_decadeindex.boxed_clone(),
options.into(),
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
height,
height_extra,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -164,7 +164,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(&mut EagerVec<Height, T>, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>,
{
@@ -186,7 +186,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
height_vec: Option<&impl AnyIterableVec<Height, T>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(height) = height_vec {
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -1,11 +1,10 @@
use std::path::Path;
use brk_error::Result;
use brk_core::{DifficultyEpoch, Height, Result, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, EagerVec, Format};
use brk_structs::{DifficultyEpoch, Height, Version};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format};
use crate::vecs::{Indexes, indexes};
use crate::{Indexes, indexes};
use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};
@@ -28,17 +27,16 @@ where
f64: From<T>,
{
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
version: Version,
format: Format,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
let height =
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?;
) -> Result<Self> {
let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)?;
let height_extra = EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -51,13 +49,13 @@ where
height,
height_extra,
difficultyepoch: EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}
@@ -68,7 +66,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(&mut EagerVec<Height, T>, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>,
{
@@ -1,20 +1,18 @@
use std::path::Path;
use brk_core::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Result, Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec,
Format, StoredIndex, VecIterator,
use brk_structs::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, Database,
EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::vecs::{
Indexes, fetched,
use crate::{
Indexes,
grouped::{ComputedVecBuilder, Source},
indexes,
indexes, price,
};
use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};
@@ -46,7 +44,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
source: Source<TxIndex, T>,
version: Version,
@@ -54,16 +52,16 @@ where
computation: Computation,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let txindex = source.is_compute().then(|| {
Box::new(
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)
.unwrap(),
)
});
let height = EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -73,7 +71,7 @@ where
let options = options.remove_percentiles();
let dateindex = EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -82,7 +80,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -93,7 +91,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -104,7 +102,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -115,7 +113,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -126,7 +124,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -137,7 +135,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -152,13 +150,13 @@ where
height,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
path,
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}
@@ -170,7 +168,7 @@ where
// starting_indexes: &Indexes,
// exit: &Exit,
// mut compute: F,
// ) -> color_eyre::Result<()>
// ) -> Result<()>
// where
// F: FnMut(
// &mut EagerVec<TxIndex, T>,
@@ -325,7 +323,7 @@ impl ComputedVecsFromTxindex<Bitcoin> {
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
.validate_computed_version_or_reset(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);
@@ -495,7 +493,7 @@ impl ComputedVecsFromTxindex<Dollars> {
exit: &Exit,
bitcoin: &ComputedVecsFromTxindex<Bitcoin>,
txindex: Option<&impl CollectableVec<TxIndex, Dollars>>,
fetched: &fetched::Vecs,
price: &price::Vecs,
) -> Result<()> {
let txindex_version = if let Some(txindex) = txindex {
txindex.version()
@@ -504,11 +502,11 @@ impl ComputedVecsFromTxindex<Dollars> {
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
.validate_computed_version_or_reset(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);
let mut close_iter = fetched.chainindexes_to_close.height.into_iter();
let mut close_iter = price.chainindexes_to_close.height.into_iter();
(starting_index.unwrap_to_usize()..indexer.vecs.height_to_weight.len())
.map(Height::from)
@@ -1,12 +1,12 @@
mod builder_computed;
mod builder_eager;
mod computed;
mod from_dateindex;
mod from_height;
mod from_height_strict;
mod from_txindex;
mod ratio_from_dateindex;
mod source;
mod r#type;
mod value_from_dateindex;
mod value_from_height;
mod value_from_txindex;
@@ -14,13 +14,13 @@ mod value_height;
pub use builder_computed::*;
pub use builder_eager::*;
use computed::*;
pub use from_dateindex::*;
pub use from_height::*;
pub use from_height_strict::*;
pub use from_txindex::*;
pub use ratio_from_dateindex::*;
pub use source::*;
use r#type::*;
pub use value_from_dateindex::*;
pub use value_from_height::*;
pub use value_from_txindex::*;
@@ -1,17 +1,12 @@
use std::{f32, path::Path};
use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, Format,
StoredIndex, VecIterator,
use brk_structs::{Date, DateIndex, Dollars, StoredF32, Version};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, Database,
EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{
utils::get_percentile,
vecs::{Indexes, fetched, grouped::source::Source, indexes},
};
use crate::{Indexes, grouped::source::Source, indexes, price, utils::get_percentile};
use super::{ComputedVecsFromDateIndex, VecBuilderOptions};
@@ -63,7 +58,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedRatioVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
source: Source<DateIndex, Dollars>,
version: Version,
@@ -71,13 +66,13 @@ impl ComputedRatioVecsFromDateIndex {
computation: Computation,
indexes: &indexes::Vecs,
extended: bool,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let options = VecBuilderOptions::default().add_last();
Ok(Self {
price: source.is_compute().then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
name,
Source::Compute,
version + VERSION,
@@ -89,7 +84,7 @@ impl ComputedRatioVecsFromDateIndex {
.unwrap()
}),
ratio: ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -100,7 +95,7 @@ impl ComputedRatioVecsFromDateIndex {
)?,
ratio_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -113,7 +108,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1w_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_1w_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -126,7 +121,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1m_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_1m_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -139,7 +134,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_1y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -152,7 +147,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_4y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -165,7 +160,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma_momentum_oscillator: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -178,7 +173,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -191,7 +186,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_4y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -204,7 +199,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_1y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -217,7 +212,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p99_9"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -230,7 +225,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p99_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -243,7 +238,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p99"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -256,7 +251,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -269,7 +264,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p0_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -282,7 +277,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p0_1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -295,7 +290,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -308,7 +303,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -321,7 +316,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -334,7 +329,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_m1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -347,7 +342,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_m2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -360,7 +355,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_m3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -373,7 +368,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p99_9_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -386,7 +381,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p99_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -399,7 +394,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p99_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -412,7 +407,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -425,7 +420,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p0_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -438,7 +433,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p0_1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -451,7 +446,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -464,7 +459,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -477,7 +472,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_p3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -490,7 +485,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_m1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -503,7 +498,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_m2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -516,7 +511,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_m3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -529,7 +524,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -542,7 +537,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_4y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -555,7 +550,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_ratio_1y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -573,11 +568,11 @@ impl ComputedRatioVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: &fetched::Vecs,
price: &price::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<DateIndex, Dollars>,
@@ -599,7 +594,7 @@ impl ComputedRatioVecsFromDateIndex {
self.compute_rest(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
date_to_price_opt,
@@ -610,16 +605,16 @@ impl ComputedRatioVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: &fetched::Vecs,
price: &price::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
date_to_price_opt: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
let date_to_price = date_to_price_opt.unwrap_or_else(|| unsafe {
std::mem::transmute(&self.price.as_ref().unwrap().dateindex)
});
let closes = fetched.timeindexes_to_close.dateindex.as_ref().unwrap();
let closes = price.timeindexes_to_close.dateindex.as_ref().unwrap();
self.ratio.compute_all(
indexer,
@@ -640,7 +635,8 @@ impl ComputedRatioVecsFromDateIndex {
}
},
exit,
)
)?;
Ok(())
},
)?;
@@ -662,7 +658,8 @@ impl ComputedRatioVecsFromDateIndex {
usize::MAX,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -678,7 +675,8 @@ impl ComputedRatioVecsFromDateIndex {
7,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -694,7 +692,8 @@ impl ComputedRatioVecsFromDateIndex {
30,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -710,7 +709,8 @@ impl ComputedRatioVecsFromDateIndex {
365,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -726,7 +726,8 @@ impl ComputedRatioVecsFromDateIndex {
4 * 365,
exit,
Some(min_ratio_date),
)
)?;
Ok(())
},
)?;
@@ -759,7 +760,8 @@ impl ComputedRatioVecsFromDateIndex {
)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -767,9 +769,10 @@ impl ComputedRatioVecsFromDateIndex {
self.mut_ratio_vecs()
.iter_mut()
.try_for_each(|v| -> Result<()> {
v.validate_computed_version_or_reset_file(
v.validate_computed_version_or_reset(
Version::ZERO + v.inner_version() + ratio_version,
)
)?;
Ok(())
})?;
let starting_dateindex = self
@@ -1072,6 +1075,10 @@ impl ComputedRatioVecsFromDateIndex {
Ok(())
})?;
drop(sma_iter);
drop(_4y_sma_iter);
drop(_1y_sma_iter);
self.mut_ratio_vecs()
.into_iter()
.try_for_each(|v| v.safe_flush(exit))?;
@@ -1193,7 +1200,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1219,7 +1227,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1245,7 +1254,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1271,7 +1281,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1297,7 +1308,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1323,7 +1335,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1349,7 +1362,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1375,7 +1389,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1401,7 +1416,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1427,7 +1443,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1453,7 +1470,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1479,7 +1497,8 @@ impl ComputedRatioVecsFromDateIndex {
(i, price * multiplier)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1495,7 +1514,8 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_sma.as_ref().unwrap().dateindex.as_ref().unwrap(),
self.ratio_sd.as_ref().unwrap().dateindex.as_ref().unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -1521,7 +1541,8 @@ impl ComputedRatioVecsFromDateIndex {
.as_ref()
.unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -1547,7 +1568,8 @@ impl ComputedRatioVecsFromDateIndex {
.as_ref()
.unwrap(),
exit,
)
)?;
Ok(())
},
)?;
@@ -1,10 +1,10 @@
use brk_vec::BoxedAnyIterableVec;
use vecdb::AnyBoxedIterableVec;
#[derive(Clone)]
pub enum Source<I, T> {
Compute,
None,
Vec(BoxedAnyIterableVec<I, T>),
Vec(AnyBoxedIterableVec<I, T>),
}
impl<I, T> Source<I, T> {
@@ -20,7 +20,7 @@ impl<I, T> Source<I, T> {
matches!(self, Self::Vec(_))
}
pub fn vec(self) -> Option<BoxedAnyIterableVec<I, T>> {
pub fn vec(self) -> Option<AnyBoxedIterableVec<I, T>> {
match self {
Self::Vec(v) => Some(v),
_ => None,
@@ -34,14 +34,14 @@ impl<I, T> From<bool> for Source<I, T> {
}
}
impl<I, T> From<BoxedAnyIterableVec<I, T>> for Source<I, T> {
fn from(value: BoxedAnyIterableVec<I, T>) -> Self {
impl<I, T> From<AnyBoxedIterableVec<I, T>> for Source<I, T> {
fn from(value: AnyBoxedIterableVec<I, T>) -> Self {
Self::Vec(value)
}
}
impl<I, T> From<Option<BoxedAnyIterableVec<I, T>>> for Source<I, T> {
fn from(value: Option<BoxedAnyIterableVec<I, T>>) -> Self {
impl<I, T> From<Option<AnyBoxedIterableVec<I, T>>> for Source<I, T> {
fn from(value: Option<AnyBoxedIterableVec<I, T>>) -> Self {
if let Some(v) = value {
Self::Vec(v)
} else {
@@ -1,11 +1,16 @@
use std::path::Path;
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
use brk_structs::{Bitcoin, DateIndex, Dollars, Sats, Version};
use vecdb::{
AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec,
};
use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
use crate::{
Indexes,
grouped::ComputedVecsFromDateIndex,
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
};
use super::{Source, VecBuilderOptions};
@@ -21,7 +26,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
source: Source<DateIndex, Sats>,
version: Version,
@@ -30,10 +35,10 @@ impl ComputedValueVecsFromDateIndex {
options: VecBuilderOptions,
compute_dollars: bool,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import(
path,
db,
name,
source,
version + VERSION,
@@ -43,7 +48,7 @@ impl ComputedValueVecsFromDateIndex {
options,
)?,
bitcoin: ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -54,7 +59,7 @@ impl ComputedValueVecsFromDateIndex {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
db,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,
@@ -72,11 +77,11 @@ impl ComputedValueVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<DateIndex, Sats>,
@@ -95,7 +100,7 @@ impl ComputedValueVecsFromDateIndex {
)?;
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, dateindex)?;
self.compute_rest(indexer, indexes, price, starting_indexes, exit, dateindex)?;
Ok(())
}
@@ -104,11 +109,11 @@ impl ComputedValueVecsFromDateIndex {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl CollectableVec<DateIndex, Sats>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(dateindex) = dateindex {
self.sats
.compute_rest(indexes, starting_indexes, exit, Some(dateindex))?;
@@ -144,7 +149,7 @@ impl ComputedValueVecsFromDateIndex {
}
let dateindex_to_bitcoin = self.bitcoin.dateindex.as_ref().unwrap();
let dateindex_to_close = fetched
let dateindex_to_close = price
.as_ref()
.unwrap()
.timeindexes_to_close
@@ -1,11 +1,16 @@
use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use vecdb::{
AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec,
};
use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
use crate::{
Indexes,
grouped::Source,
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
};
use super::{ComputedVecsFromHeight, VecBuilderOptions};
@@ -21,7 +26,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromHeight {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
source: Source<Height, Sats>,
version: Version,
@@ -30,10 +35,10 @@ impl ComputedValueVecsFromHeight {
options: VecBuilderOptions,
compute_dollars: bool,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromHeight::forced_import(
path,
db,
name,
source,
version + VERSION,
@@ -43,7 +48,7 @@ impl ComputedValueVecsFromHeight {
options,
)?,
bitcoin: ComputedVecsFromHeight::forced_import(
path,
db,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -54,7 +59,7 @@ impl ComputedValueVecsFromHeight {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
db,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,
@@ -72,11 +77,11 @@ impl ComputedValueVecsFromHeight {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<Height, Sats>,
@@ -95,7 +100,7 @@ impl ComputedValueVecsFromHeight {
)?;
let height: Option<&StoredVec<Height, Sats>> = None;
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, height)?;
self.compute_rest(indexer, indexes, price, starting_indexes, exit, height)?;
Ok(())
}
@@ -104,11 +109,11 @@ impl ComputedValueVecsFromHeight {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&impl CollectableVec<Height, Sats>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(height) = height {
self.sats
.compute_rest(indexes, starting_indexes, exit, Some(height))?;
@@ -144,7 +149,7 @@ impl ComputedValueVecsFromHeight {
}
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
let height_to_close = &price.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_all(
@@ -1,14 +1,12 @@
use std::path::Path;
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3,
Format, LazyVecFrom1, StoredIndex, StoredVec,
use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Computation, ComputedVecFrom3,
Database, Exit, Format, LazyVecFrom1, StoredIndex, StoredVec,
};
use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
use crate::{Indexes, grouped::Source, indexes, price};
use super::{ComputedVecsFromTxindex, VecBuilderOptions};
@@ -38,23 +36,23 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromTxindex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
indexes: &indexes::Vecs,
source: Source<TxIndex, Sats>,
version: Version,
computation: Computation,
format: Format,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
) -> Result<Self> {
let compute_dollars = price.is_some();
let name_in_btc = format!("{name}_in_btc");
let name_in_usd = format!("{name}_in_usd");
let sats = ComputedVecsFromTxindex::forced_import(
path,
db,
name,
source.clone(),
version + VERSION,
@@ -79,7 +77,7 @@ impl ComputedValueVecsFromTxindex {
);
let bitcoin = ComputedVecsFromTxindex::forced_import(
path,
db,
&name_in_btc,
Source::None,
version + VERSION,
@@ -89,16 +87,16 @@ impl ComputedValueVecsFromTxindex {
options,
)?;
let dollars_txindex = fetched.map(|fetched| {
let dollars_txindex = price.map(|price| {
ComputedVecFrom3::forced_import_or_init_from_3(
computation,
path,
db,
&name_in_usd,
version + VERSION,
format,
bitcoin_txindex.boxed_clone(),
indexes.txindex_to_height.boxed_clone(),
fetched.chainindexes_to_close.height.boxed_clone(),
price.chainindexes_to_close.height.boxed_clone(),
|txindex: TxIndex,
txindex_to_btc_iter,
txindex_to_height_iter,
@@ -127,7 +125,7 @@ impl ComputedValueVecsFromTxindex {
dollars_txindex,
dollars: compute_dollars.then(|| {
ComputedVecsFromTxindex::forced_import(
path,
db,
&name_in_usd,
Source::None,
version + VERSION,
@@ -145,11 +143,11 @@ impl ComputedValueVecsFromTxindex {
// &mut self,
// indexer: &Indexer,
// indexes: &indexes::Vecs,
// fetched: Option<&marketprice::Vecs>,
// price: Option<&marketprice::Vecs>,
// starting_indexes: &Indexes,
// exit: &Exit,
// mut compute: F,
// ) -> color_eyre::Result<()>
// ) -> Result<()>
// where
// F: FnMut(
// &mut EagerVec<TxIndex, Sats>,
@@ -187,8 +185,8 @@ impl ComputedValueVecsFromTxindex {
starting_indexes: &Indexes,
exit: &Exit,
txindex: Option<&impl CollectableVec<TxIndex, Sats>>,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<()> {
price: Option<&price::Vecs>,
) -> Result<()> {
if let Some(txindex) = txindex {
self.sats
.compute_rest(indexer, indexes, starting_indexes, exit, Some(txindex))?;
@@ -223,7 +221,7 @@ impl ComputedValueVecsFromTxindex {
exit,
&self.bitcoin,
Some(dollars_txindex),
fetched.as_ref().unwrap(),
price.as_ref().unwrap(),
)?;
}
@@ -1,11 +1,14 @@
use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use vecdb::{AnyCollectableVec, CollectableVec, Database, EagerVec, Exit, Format, StoredVec};
use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
use crate::{
Indexes,
grouped::Source,
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
};
#[derive(Clone)]
pub struct ComputedHeightValueVecs {
@@ -18,27 +21,27 @@ const VERSION: Version = Version::ZERO;
impl ComputedHeightValueVecs {
pub fn forced_import(
path: &Path,
db: &Database,
name: &str,
source: Source<Height, Sats>,
version: Version,
format: Format,
compute_dollars: bool,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self {
sats: source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)
.unwrap()
}),
bitcoin: EagerVec::forced_import(
path,
db,
&format!("{name}_in_btc"),
version + VERSION + Version::ZERO,
format,
)?,
dollars: compute_dollars.then(|| {
EagerVec::forced_import(
path,
db,
&format!("{name}_in_usd"),
version + VERSION + Version::ZERO,
format,
@@ -52,11 +55,11 @@ impl ComputedHeightValueVecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
) -> Result<()>
where
F: FnMut(
&mut EagerVec<Height, Sats>,
@@ -75,18 +78,18 @@ impl ComputedHeightValueVecs {
)?;
let height: Option<&StoredVec<Height, Sats>> = None;
self.compute_rest(fetched, starting_indexes, exit, height)?;
self.compute_rest(price, starting_indexes, exit, height)?;
Ok(())
}
pub fn compute_rest(
&mut self,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&impl CollectableVec<Height, Sats>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(height) = height {
self.bitcoin
.compute_from_sats(starting_indexes.height, height, exit)?;
@@ -99,7 +102,7 @@ impl ComputedHeightValueVecs {
}
let height_to_bitcoin = &self.bitcoin;
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
let height_to_close = &price.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_from_bitcoin(
@@ -1,38 +1,38 @@
use std::{ops::Deref, path::Path};
use brk_core::{
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2AAddressIndex, P2ABytes, P2MSOutputIndex,
P2PK33AddressIndex, P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex,
P2PKHBytes, P2SHAddressIndex, P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex,
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredUsize,
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64,
Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator,
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, Database, EagerVec, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator,
};
use crate::vecs::indexes;
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub dateindex_to_date: EagerVec<DateIndex, Date>,
pub dateindex_to_dateindex: EagerVec<DateIndex, DateIndex>,
pub dateindex_to_first_height: EagerVec<DateIndex, Height>,
pub dateindex_to_height_count: EagerVec<DateIndex, StoredUsize>,
pub dateindex_to_height_count: EagerVec<DateIndex, StoredU64>,
pub dateindex_to_monthindex: EagerVec<DateIndex, MonthIndex>,
pub dateindex_to_weekindex: EagerVec<DateIndex, WeekIndex>,
pub decadeindex_to_decadeindex: EagerVec<DecadeIndex, DecadeIndex>,
pub decadeindex_to_first_yearindex: EagerVec<DecadeIndex, YearIndex>,
pub decadeindex_to_yearindex_count: EagerVec<DecadeIndex, StoredUsize>,
pub decadeindex_to_yearindex_count: EagerVec<DecadeIndex, StoredU64>,
pub difficultyepoch_to_difficultyepoch: EagerVec<DifficultyEpoch, DifficultyEpoch>,
pub difficultyepoch_to_first_height: EagerVec<DifficultyEpoch, Height>,
pub difficultyepoch_to_height_count: EagerVec<DifficultyEpoch, StoredUsize>,
pub difficultyepoch_to_height_count: EagerVec<DifficultyEpoch, StoredU64>,
pub emptyoutputindex_to_emptyoutputindex:
ComputedVecFrom1<EmptyOutputIndex, EmptyOutputIndex, EmptyOutputIndex, TxIndex>,
pub halvingepoch_to_first_height: EagerVec<HalvingEpoch, Height>,
@@ -44,9 +44,9 @@ pub struct Vecs {
pub height_to_halvingepoch: EagerVec<Height, HalvingEpoch>,
pub height_to_height: EagerVec<Height, Height>,
pub height_to_timestamp_fixed: EagerVec<Height, Timestamp>,
pub height_to_txindex_count: EagerVec<Height, StoredUsize>,
pub height_to_txindex_count: EagerVec<Height, StoredU64>,
pub inputindex_to_inputindex: ComputedVecFrom1<InputIndex, InputIndex, InputIndex, OutputIndex>,
pub monthindex_to_dateindex_count: EagerVec<MonthIndex, StoredUsize>,
pub monthindex_to_dateindex_count: EagerVec<MonthIndex, StoredU64>,
pub monthindex_to_first_dateindex: EagerVec<MonthIndex, DateIndex>,
pub monthindex_to_monthindex: EagerVec<MonthIndex, MonthIndex>,
pub monthindex_to_quarterindex: EagerVec<MonthIndex, QuarterIndex>,
@@ -75,39 +75,42 @@ pub struct Vecs {
pub p2wshaddressindex_to_p2wshaddressindex:
ComputedVecFrom1<P2WSHAddressIndex, P2WSHAddressIndex, P2WSHAddressIndex, P2WSHBytes>,
pub quarterindex_to_first_monthindex: EagerVec<QuarterIndex, MonthIndex>,
pub quarterindex_to_monthindex_count: EagerVec<QuarterIndex, StoredUsize>,
pub quarterindex_to_monthindex_count: EagerVec<QuarterIndex, StoredU64>,
pub quarterindex_to_quarterindex: EagerVec<QuarterIndex, QuarterIndex>,
pub semesterindex_to_first_monthindex: EagerVec<SemesterIndex, MonthIndex>,
pub semesterindex_to_monthindex_count: EagerVec<SemesterIndex, StoredUsize>,
pub semesterindex_to_monthindex_count: EagerVec<SemesterIndex, StoredU64>,
pub semesterindex_to_semesterindex: EagerVec<SemesterIndex, SemesterIndex>,
pub txindex_to_height: EagerVec<TxIndex, Height>,
pub txindex_to_input_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, InputIndex, InputIndex, OutputIndex>,
ComputedVecFrom2<TxIndex, StoredU64, TxIndex, InputIndex, InputIndex, OutputIndex>,
pub txindex_to_output_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, OutputIndex, OutputIndex, Sats>,
ComputedVecFrom2<TxIndex, StoredU64, TxIndex, OutputIndex, OutputIndex, Sats>,
pub txindex_to_txindex: ComputedVecFrom1<TxIndex, TxIndex, TxIndex, Txid>,
pub unknownoutputindex_to_unknownoutputindex:
ComputedVecFrom1<UnknownOutputIndex, UnknownOutputIndex, UnknownOutputIndex, TxIndex>,
pub weekindex_to_dateindex_count: EagerVec<WeekIndex, StoredUsize>,
pub weekindex_to_dateindex_count: EagerVec<WeekIndex, StoredU64>,
pub weekindex_to_first_dateindex: EagerVec<WeekIndex, DateIndex>,
pub weekindex_to_weekindex: EagerVec<WeekIndex, WeekIndex>,
pub yearindex_to_decadeindex: EagerVec<YearIndex, DecadeIndex>,
pub yearindex_to_first_monthindex: EagerVec<YearIndex, MonthIndex>,
pub yearindex_to_monthindex_count: EagerVec<YearIndex, StoredUsize>,
pub yearindex_to_monthindex_count: EagerVec<YearIndex, StoredU64>,
pub yearindex_to_yearindex: EagerVec<YearIndex, YearIndex>,
}
impl Vecs {
pub fn forced_import(
path: &Path,
parent: &Path,
version: Version,
indexer: &Indexer,
computation: Computation,
format: Format,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let db = Database::open(&parent.join("indexes"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"outputindex",
version + VERSION + Version::ZERO,
format,
@@ -117,7 +120,7 @@ impl Vecs {
let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"inputindex",
version + VERSION + Version::ZERO,
format,
@@ -127,7 +130,7 @@ impl Vecs {
let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"txindex",
version + VERSION + Version::ZERO,
format,
@@ -137,7 +140,7 @@ impl Vecs {
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
&db,
"input_count",
version + VERSION + Version::ZERO,
format,
@@ -153,14 +156,14 @@ impl Vecs {
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_owned()))
.unwrap_or_else(|| inputindex_to_outputindex_iter.len());
StoredUsize::from((start..end).count())
StoredU64::from((start..end).count())
})
},
)?;
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
&db,
"output_count",
version + VERSION + Version::ZERO,
format,
@@ -176,14 +179,14 @@ impl Vecs {
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_owned()))
.unwrap_or_else(|| outputindex_to_value_iter.len());
StoredUsize::from((start..end).count())
StoredU64::from((start..end).count())
})
},
)?;
let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2pk33addressindex",
version + VERSION + Version::ZERO,
format,
@@ -192,7 +195,7 @@ impl Vecs {
)?;
let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2pk65addressindex",
version + VERSION + Version::ZERO,
format,
@@ -201,7 +204,7 @@ impl Vecs {
)?;
let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2pkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -210,7 +213,7 @@ impl Vecs {
)?;
let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2shaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -219,7 +222,7 @@ impl Vecs {
)?;
let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2traddressindex",
version + VERSION + Version::ZERO,
format,
@@ -228,7 +231,7 @@ impl Vecs {
)?;
let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2wpkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -237,7 +240,7 @@ impl Vecs {
)?;
let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2wshaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -246,7 +249,7 @@ impl Vecs {
)?;
let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2aaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -255,7 +258,7 @@ impl Vecs {
)?;
let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"p2msoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -264,7 +267,7 @@ impl Vecs {
)?;
let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"emptyoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -273,7 +276,7 @@ impl Vecs {
)?;
let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"unknownoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -282,7 +285,7 @@ impl Vecs {
)?;
let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"opreturnindex",
version + VERSION + Version::ZERO,
format,
@@ -310,263 +313,265 @@ impl Vecs {
unknownoutputindex_to_unknownoutputindex,
dateindex_to_date: EagerVec::forced_import(
path,
&db,
"date",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
path,
&db,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_first_height: EagerVec::forced_import(
path,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_monthindex: EagerVec::forced_import(
path,
&db,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_weekindex: EagerVec::forced_import(
path,
&db,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_decadeindex: EagerVec::forced_import(
path,
&db,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_first_yearindex: EagerVec::forced_import(
path,
&db,
"first_yearindex",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
path,
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_first_height: EagerVec::forced_import(
path,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_first_height: EagerVec::forced_import(
path,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_halvingepoch: EagerVec::forced_import(
path,
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date: EagerVec::forced_import(
path,
&db,
"date",
version + VERSION + Version::ZERO,
format,
)?,
height_to_difficultyepoch: EagerVec::forced_import(
path,
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_halvingepoch: EagerVec::forced_import(
path,
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_height: EagerVec::forced_import(
path,
&db,
"height",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_first_dateindex: EagerVec::forced_import(
path,
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_monthindex: EagerVec::forced_import(
path,
&db,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_quarterindex: EagerVec::forced_import(
path,
&db,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_semesterindex: EagerVec::forced_import(
path,
&db,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_yearindex: EagerVec::forced_import(
path,
&db,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_first_monthindex: EagerVec::forced_import(
path,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_first_monthindex: EagerVec::forced_import(
path,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_first_dateindex: EagerVec::forced_import(
path,
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_first_monthindex: EagerVec::forced_import(
path,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_quarterindex: EagerVec::forced_import(
path,
&db,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_semesterindex: EagerVec::forced_import(
path,
&db,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_weekindex: EagerVec::forced_import(
path,
&db,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_decadeindex: EagerVec::forced_import(
path,
&db,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_yearindex: EagerVec::forced_import(
path,
&db,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date_fixed: EagerVec::forced_import(
path,
&db,
"date_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_dateindex: EagerVec::forced_import(
path,
&db,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_height: EagerVec::forced_import(
path,
&db,
"height",
version + VERSION + Version::ZERO,
format,
)?,
height_to_timestamp_fixed: EagerVec::forced_import(
path,
&db,
"timestamp_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_txindex_count: EagerVec::forced_import(
path,
&db,
"txindex_count",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_height_count: EagerVec::forced_import(
path,
&db,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_dateindex_count: EagerVec::forced_import(
path,
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_height_count: EagerVec::forced_import(
path,
&db,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_dateindex_count: EagerVec::forced_import(
path,
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_monthindex_count: EagerVec::forced_import(
path,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_monthindex_count: EagerVec::forced_import(
path,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_monthindex_count: EagerVec::forced_import(
path,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_yearindex_count: EagerVec::forced_import(
path,
&db,
"yearindex_count",
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_txindex: EagerVec::forced_import(
path,
&db,
"txindex",
version + VERSION + Version::ZERO,
format,
)?,
db,
})
}
@@ -575,7 +580,18 @@ impl Vecs {
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> color_eyre::Result<Indexes> {
) -> Result<Indexes> {
let idxs = self.compute_(indexer, starting_indexes, exit)?;
self.db.flush_then_punch()?;
Ok(idxs)
}
fn compute_(
&mut self,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> Result<Indexes> {
// ---
// OutputIndex
// ---
@@ -744,14 +760,14 @@ impl Vecs {
starting_indexes.height,
&indexer.vecs.height_to_timestamp,
|(h, timestamp, height_to_timestamp_fixed_iter)| {
if prev_timestamp_fixed.is_none() {
if let Some(prev_h) = h.decremented() {
prev_timestamp_fixed.replace(
height_to_timestamp_fixed_iter
.into_iter()
.unwrap_get_inner(prev_h),
);
}
if prev_timestamp_fixed.is_none()
&& let Some(prev_h) = h.decremented()
{
prev_timestamp_fixed.replace(
height_to_timestamp_fixed_iter
.into_iter()
.unwrap_get_inner(prev_h),
);
}
let timestamp_fixed =
prev_timestamp_fixed.map_or(timestamp, |prev_d| prev_d.max(timestamp));
@@ -1206,7 +1222,7 @@ pub struct Indexes {
}
impl Indexes {
pub fn update_from_height(&mut self, height: Height, indexes: &indexes::Vecs) {
pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
self.indexes.height = height;
self.dateindex = DateIndex::try_from(
indexes
+205 -37
View File
@@ -1,31 +1,48 @@
#![doc = include_str!("../README.md")]
#![doc = "\n## Example\n\n```rust"]
#![doc = include_str!("../examples/main.rs")]
#![doc = include_str!("../examples/computer.rs")]
#![doc = "```"]
use std::path::Path;
use brk_core::Version;
use brk_exit::Exit;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{Computation, Format};
use brk_structs::Version;
use log::info;
use vecdb::{AnyCollectableVec, Computation, Exit, Format};
mod blocks;
mod cointime;
mod constants;
mod fetched;
mod grouped;
mod indexes;
mod market;
mod mining;
mod price;
mod stateful;
mod states;
mod stores;
mod traits;
mod transactions;
mod utils;
mod vecs;
use indexes::Indexes;
use states::*;
use stores::Stores;
use vecs::Vecs;
#[derive(Clone)]
pub struct Computer {
fetcher: Option<Fetcher>,
pub vecs: Vecs,
pub stores: Stores,
pub indexes: indexes::Vecs,
pub constants: constants::Vecs,
pub blocks: blocks::Vecs,
pub mining: mining::Vecs,
pub market: market::Vecs,
pub price: Option<price::Vecs>,
pub transactions: transactions::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
pub cointime: cointime::Vecs,
}
const VERSION: Version = Version::ONE;
@@ -33,47 +50,198 @@ const VERSION: Version = Version::ONE;
impl Computer {
/// Do NOT import multiple times or things will break !!!
pub fn forced_import(
outputs_dir: &Path,
outputs_path: &Path,
indexer: &Indexer,
computation: Computation,
fetcher: Option<Fetcher>,
format: Format,
) -> color_eyre::Result<Self> {
Ok(Self {
vecs: Vecs::import(
// TODO: Give self.path, join inside import
&outputs_dir.join("vecs/computed"),
) -> Result<Self> {
let computed_path = outputs_path.join("computed");
let computation = Computation::Lazy;
let format = Format::Compressed;
let indexes = indexes::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
indexer,
computation,
format,
)?;
let fetched = fetcher.map(|fetcher| {
fetched::Vecs::forced_import(outputs_path, fetcher, VERSION + Version::ZERO).unwrap()
});
let price = fetched.is_some().then(|| {
price::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
indexer,
fetcher.is_some(),
computation,
format,
)?,
stores: Stores::import(
// TODO: Give self.path, join inside import
&outputs_dir.join("stores"),
&indexes,
)
.unwrap()
});
Ok(Self {
blocks: blocks::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
&indexer.stores.keyspace,
computation,
format,
&indexes,
)?,
fetcher,
mining: mining::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
constants: constants::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
market: market::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
stateful: stateful::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
price.as_ref(),
&computed_path.join("states"),
)?,
transactions: transactions::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
indexer,
&indexes,
computation,
format,
price.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
computation,
format,
&indexes,
price.as_ref(),
)?,
indexes,
fetched,
price,
})
}
}
impl Computer {
pub fn compute(
&mut self,
indexer: &mut Indexer,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
info!("Computing...");
self.vecs.compute(
) -> Result<()> {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
self.constants
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing blocks...");
self.blocks
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing mining...");
self.mining
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing prices...");
self.price.as_mut().unwrap().compute(
indexer,
&self.indexes,
&starting_indexes,
fetched,
exit,
)?;
}
info!("Computing transactions...");
self.transactions.compute(
indexer,
starting_indexes,
self.fetcher.as_mut(),
&self.indexes,
&starting_indexes,
self.price.as_ref(),
exit,
&mut self.stores,
)
)?;
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market.compute(
indexer,
&self.indexes,
price,
&mut self.transactions,
&starting_indexes,
exit,
)?;
}
info!("Computing stateful...");
self.stateful.compute(
indexer,
&self.indexes,
&self.transactions,
self.price.as_ref(),
&self.market,
&mut starting_indexes,
exit,
)?;
self.cointime.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
&self.transactions,
&self.stateful,
exit,
)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.constants.vecs(),
self.indexes.vecs(),
self.blocks.vecs(),
self.mining.vecs(),
self.market.vecs(),
self.transactions.vecs(),
self.stateful.vecs(),
self.cointime.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
self.price.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
pub fn static_clone(&self) -> &'static Self {
Box::leak(Box::new(self.clone()))
}
}
File diff suppressed because it is too large Load Diff
@@ -1,11 +1,11 @@
use std::path::Path;
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator};
use crate::vecs::grouped::Source;
use crate::grouped::Source;
use super::{
Indexes,
@@ -17,6 +17,8 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex<DifficultyEpoch>,
pub indexes_to_halvingepoch: ComputedVecsFromDateIndex<HalvingEpoch>,
@@ -24,15 +26,18 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
parent: &Path,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
let db = Database::open(&parent.join("mining"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
path,
&db,
"difficulty",
Source::None,
version + VERSION + Version::ZERO,
@@ -42,7 +47,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
path,
&db,
"difficultyepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -52,7 +57,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
path,
&db,
"halvingepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -61,6 +66,8 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_last(),
)?,
db,
})
}
@@ -70,7 +77,19 @@ impl Vecs {
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
self.indexes_to_difficultyepoch.compute_all(
indexer,
@@ -91,7 +110,8 @@ impl Vecs {
)
},
exit,
)
)?;
Ok(())
},
)?;
@@ -115,7 +135,8 @@ impl Vecs {
)
},
exit,
)
)?;
Ok(())
},
)?;
File diff suppressed because it is too large Load Diff
@@ -1,23 +1,22 @@
use std::{ops::Deref, path::Path};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator,
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, Database, EagerVec, Exit,
Format, GenericStoredVec, VecIterator,
};
use crate::{
states::AddressCohortState,
vecs::{
Indexes, fetched,
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, market,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
},
Indexes,
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, market, price,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
},
states::AddressCohortState,
};
const VERSION: Version = Version::ZERO;
@@ -26,46 +25,49 @@ const VERSION: Version = Version::ZERO;
pub struct Vecs {
starting_height: Height,
pub state: AddressCohortState,
pub state: Option<AddressCohortState>,
pub inner: common::Vecs,
pub height_to_address_count: EagerVec<Height, StoredUsize>,
pub indexes_to_address_count: ComputedVecsFromHeight<StoredUsize>,
pub height_to_address_count: EagerVec<Height, StoredU64>,
pub indexes_to_address_count: ComputedVecsFromHeight<StoredU64>,
}
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
version: Version,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
states_path: &Path,
price: Option<&price::Vecs>,
states_path: Option<&Path>,
compute_relative_to_all: bool,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
) -> Result<Self> {
let compute_dollars = price.is_some();
let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}"));
Ok(Self {
starting_height: Height::ZERO,
state: AddressCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)?,
state: states_path.map(|states_path| {
AddressCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)
.unwrap()
}),
height_to_address_count: EagerVec::forced_import(
path,
db,
&suffix("address_count"),
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
path,
db,
&suffix("address_count"),
Source::None,
version + VERSION + Version::ZERO,
@@ -75,15 +77,16 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
inner: common::Vecs::forced_import(
path,
db,
cohort_name,
computation,
format,
version,
indexes,
fetched,
price,
compute_relative_to_all,
false,
false,
)?,
})
}
@@ -92,7 +95,9 @@ impl Vecs {
impl DynCohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.state.as_ref().map_or(Height::MAX, |state| {
state.height().map_or(Height::MAX, |h| h.incremented())
}),
self.height_to_address_count.len().into(),
self.inner.starting_height(),
]
@@ -109,19 +114,21 @@ impl DynCohortVecs for Vecs {
self.starting_height = starting_height;
if let Some(prev_height) = starting_height.decremented() {
self.state.address_count = *self
self.state.as_mut().unwrap().address_count = *self
.height_to_address_count
.into_iter()
.unwrap_get_inner(prev_height);
}
self.inner
.init(&mut self.starting_height, &mut self.state.inner);
self.inner.init(
&mut self.starting_height,
&mut self.state.as_mut().unwrap().inner,
);
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.height_to_address_count
.validate_computed_version_or_reset_file(
.validate_computed_version_or_reset(
base_version + self.height_to_address_count.inner_version(),
)?;
@@ -135,11 +142,12 @@ impl DynCohortVecs for Vecs {
self.height_to_address_count.forced_push_at(
height,
self.state.address_count.into(),
self.state.as_ref().unwrap().address_count.into(),
exit,
)?;
self.inner.forced_pushed_at(height, exit, &self.state.inner)
self.inner
.forced_pushed_at(height, exit, &self.state.as_ref().unwrap().inner)
}
fn compute_then_force_push_unrealized_states(
@@ -156,7 +164,7 @@ impl DynCohortVecs for Vecs {
dateindex,
date_price,
exit,
&self.state.inner,
&self.state.as_ref().unwrap().inner,
)
}
@@ -164,7 +172,7 @@ impl DynCohortVecs for Vecs {
self.height_to_address_count.safe_flush(exit)?;
self.inner
.safe_flush_stateful_vecs(height, exit, &mut self.state.inner)
.safe_flush_stateful_vecs(height, exit, &mut self.state.as_mut().unwrap().inner)
}
#[allow(clippy::too_many_arguments)]
@@ -172,10 +180,10 @@ impl DynCohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.indexes_to_address_count.compute_rest(
indexes,
starting_indexes,
@@ -184,7 +192,7 @@ impl DynCohortVecs for Vecs {
)?;
self.inner
.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)
}
fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -225,7 +233,7 @@ impl CohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -233,11 +241,11 @@ impl CohortVecs for Vecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.inner.compute_rest_part2(
indexer,
indexes,
fetched,
price,
starting_indexes,
market,
height_to_supply,
@@ -1,16 +1,16 @@
use std::path::Path;
use brk_core::{
AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, GroupFilter, Height,
Result, Version,
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
Dollars, GroupFilter, Height, Version,
};
use brk_exit::Exit;
use brk_vec::{Computation, Format};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{AnyIterableVec, Computation, Database, Exit, Format};
use crate::vecs::{
Indexes, fetched, indexes,
use crate::{
Indexes, indexes, market, price,
stateful::{
address_cohort,
r#trait::{CohortVecs, DynCohortVecs},
@@ -24,470 +24,470 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>);
impl Vecs {
pub fn forced_import(
path: &Path,
db: &Database,
version: Version,
_computation: Computation,
format: Format,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
states_path: &Path,
) -> color_eyre::Result<Self> {
) -> Result<Self> {
Ok(Self(
AddressGroups {
amount_range: ByAmountRange {
_0sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_with_0sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_1sat_to_10sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1sat_under_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_10sats_to_100sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10sats_under_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_100sats_to_1k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_100sats_under_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_1k_sats_to_10k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1k_sats_under_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_10k_sats_to_100k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10k_sats_under_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_100k_sats_to_1m_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_100k_sats_under_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_1m_sats_to_10m_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1m_sats_under_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_10m_sats_to_1btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10m_sats_under_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_1btc_to_10btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1btc_under_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_10btc_to_100btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10btc_under_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_100btc_to_1k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_100btc_under_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_1k_btc_to_10k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1k_btc_under_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_10k_btc_to_100k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10k_btc_under_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
_100k_btc_or_more: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
Some(states_path),
true,
)?,
},
lt_amount: ByLowerThanAmount {
_10sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_100k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_under_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
},
ge_amount: ByGreatEqualAmount {
_1sat: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1sat"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
path,
db,
Some("addrs_above_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
fetched,
states_path,
price,
None,
true,
)?,
},
@@ -535,16 +535,61 @@ impl Vecs {
})
.collect::<Vec<_>>(),
]
.into_par_iter()
.into_iter()
.flatten()
.try_for_each(|(vecs, stateful)| {
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
})
}
pub fn compute_rest_part1(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.as_mut_vecs().into_iter().try_for_each(|(_, v)| {
v.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)
})
}
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl AnyIterableVec<DateIndex, Bitcoin>,
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.0.as_boxed_mut_vecs().into_iter().try_for_each(|v| {
v.into_iter().try_for_each(|(_, v)| {
v.compute_rest_part2(
indexer,
indexes,
price,
starting_indexes,
market,
height_to_supply,
dateindex_to_supply,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
})
})
}
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.as_mut_separate_vecs()
.par_iter_mut()
.into_iter()
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))
}
}
@@ -1,11 +1,11 @@
use brk_core::{ByAddressType, Height};
use brk_vec::VecIterator;
use brk_structs::{ByAddressType, Height};
use derive_deref::{Deref, DerefMut};
use vecdb::VecIterator;
use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(ByAddressType<usize>);
pub struct AddressTypeToAddressCount(ByAddressType<u64>);
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
@@ -1,15 +1,15 @@
use brk_core::{ByAddressType, Height, Result, StoredUsize};
use brk_exit::Exit;
use brk_vec::EagerVec;
use brk_error::Result;
use brk_structs::{ByAddressType, Height, StoredU64};
use vecdb::{EagerVec, Exit, GenericStoredVec};
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredUsize>>);
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredU64>>);
impl From<ByAddressType<EagerVec<Height, StoredUsize>>> for AddressTypeToHeightToAddressCount {
fn from(value: ByAddressType<EagerVec<Height, StoredUsize>>) -> Self {
impl From<ByAddressType<EagerVec<Height, StoredU64>>> for AddressTypeToHeightToAddressCount {
fn from(value: ByAddressType<EagerVec<Height, StoredU64>>) -> Self {
Self(value)
}
}
@@ -1,20 +1,18 @@
use brk_core::{ByAddressType, StoredUsize};
use brk_exit::Exit;
use brk_vec::AnyCollectableVec;
use brk_error::Result;
use brk_structs::{ByAddressType, StoredU64};
use derive_deref::{Deref, DerefMut};
use vecdb::{AnyCollectableVec, Exit};
use crate::vecs::{
use crate::{
Indexes, grouped::ComputedVecsFromHeight, indexes,
stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
};
#[derive(Clone, Deref, DerefMut)]
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredUsize>>);
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredU64>>);
impl From<ByAddressType<ComputedVecsFromHeight<StoredUsize>>>
for AddressTypeToIndexesToAddressCount
{
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredUsize>>) -> Self {
impl From<ByAddressType<ComputedVecsFromHeight<StoredU64>>> for AddressTypeToIndexesToAddressCount {
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredU64>>) -> Self {
Self(value)
}
}
@@ -27,7 +25,7 @@ impl AddressTypeToIndexesToAddressCount {
starting_indexes: &Indexes,
exit: &Exit,
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.p2pk65.compute_rest(
indexes,
starting_indexes,
@@ -1,6 +1,6 @@
use std::{collections::BTreeMap, mem};
use brk_core::TypeIndex;
use brk_structs::TypeIndex;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
@@ -29,10 +29,6 @@ impl<T> AddressTypeToTypeIndexTree<T> {
mem::swap(own, other);
}
}
pub fn unwrap(self) -> ByAddressType<BTreeMap<TypeIndex, T>> {
self.0
}
}
impl<T> Default for AddressTypeToTypeIndexTree<T> {
@@ -0,0 +1,57 @@
use std::mem;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
impl<T> AddressTypeToVec<T> {
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
self
}
pub fn merge_mut(&mut self, mut other: Self) {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
}
fn merge_(own: &mut Vec<T>, other: &mut Vec<T>) {
if own.len() >= other.len() {
own.append(other);
} else {
other.append(own);
mem::swap(own, other);
}
}
}
impl<T> Default for AddressTypeToVec<T> {
fn default() -> Self {
Self(ByAddressType {
p2pk65: vec![],
p2pk33: vec![],
p2pkh: vec![],
p2sh: vec![],
p2wpkh: vec![],
p2wsh: vec![],
p2tr: vec![],
p2a: vec![],
})
}
}
@@ -1,9 +1,9 @@
use std::collections::BTreeMap;
use brk_core::Height;
use brk_structs::Height;
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::AddressTypeToVec;
use crate::stateful::AddressTypeToVec;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);
File diff suppressed because it is too large Load Diff
@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_vec::{IndexedVec, StoredIndex, StoredType};
use vecdb::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw};
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
@@ -20,12 +20,26 @@ where
}
}
impl<I, T> From<&IndexedVec<I, T>> for RangeMap<T, I>
impl<I, T> From<&RawVec<I, T>> for RangeMap<T, I>
where
I: StoredIndex,
T: StoredIndex + StoredType,
T: StoredIndex + StoredRaw,
{
fn from(vec: &IndexedVec<I, T>) -> Self {
fn from(vec: &RawVec<I, T>) -> Self {
Self(
vec.into_iter()
.map(|(i, v)| (v.into_owned(), i))
.collect::<BTreeMap<_, _>>(),
)
}
}
impl<I, T> From<&CompressedVec<I, T>> for RangeMap<T, I>
where
I: StoredIndex,
T: StoredIndex + StoredCompressed,
{
fn from(vec: &CompressedVec<I, T>) -> Self {
Self(
vec.into_iter()
.map(|(i, v)| (v.into_owned(), i))
@@ -1,9 +1,9 @@
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec};
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{AnyCollectableVec, AnyIterableVec, Exit};
use crate::vecs::{Indexes, fetched, indexes, market};
use crate::{Indexes, indexes, market, price};
pub trait DynCohortVecs: Send + Sync {
fn starting_height(&self) -> Height;
@@ -30,10 +30,10 @@ pub trait DynCohortVecs: Send + Sync {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()>;
) -> Result<()>;
fn vecs(&self) -> Vec<&dyn AnyCollectableVec>;
}
@@ -51,7 +51,7 @@ pub trait CohortVecs: DynCohortVecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -59,5 +59,5 @@ pub trait CohortVecs: DynCohortVecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()>;
) -> Result<()>;
}
@@ -1,18 +1,15 @@
use std::{ops::Deref, path::Path};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format};
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{AnyCollectableVec, AnyIterableVec, Computation, Database, Exit, Format};
use crate::{
UTXOCohortState,
vecs::{
Indexes, fetched, indexes, market,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
},
Indexes, UTXOCohortState, indexes, market, price,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
},
};
@@ -20,7 +17,7 @@ use crate::{
pub struct Vecs {
starting_height: Height,
pub state: UTXOCohortState,
pub state: Option<UTXOCohortState>,
inner: common::Vecs,
}
@@ -28,38 +25,43 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
db: &Database,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
version: Version,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
states_path: &Path,
price: Option<&price::Vecs>,
states_path: Option<&Path>,
compute_relative_to_all: bool,
ratio_extended: bool,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
compute_adjusted: bool,
) -> Result<Self> {
let compute_dollars = price.is_some();
Ok(Self {
starting_height: Height::ZERO,
state: UTXOCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)?,
state: states_path.map(|states_path| {
UTXOCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)
.unwrap()
}),
inner: common::Vecs::forced_import(
path,
db,
cohort_name,
computation,
format,
version,
indexes,
fetched,
price,
compute_relative_to_all,
ratio_extended,
compute_adjusted,
)?,
})
}
@@ -68,7 +70,9 @@ impl Vecs {
impl DynCohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.state.as_ref().map_or(Height::MAX, |state| {
state.height().map_or(Height::MAX, |h| h.incremented())
}),
self.inner.starting_height(),
]
.into_iter()
@@ -83,7 +87,8 @@ impl DynCohortVecs for Vecs {
self.starting_height = starting_height;
self.inner.init(&mut self.starting_height, &mut self.state);
self.inner
.init(&mut self.starting_height, self.state.as_mut().unwrap());
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
@@ -95,7 +100,8 @@ impl DynCohortVecs for Vecs {
return Ok(());
}
self.inner.forced_pushed_at(height, exit, &self.state)
self.inner
.forced_pushed_at(height, exit, self.state.as_ref().unwrap())
}
fn compute_then_force_push_unrealized_states(
@@ -112,13 +118,13 @@ impl DynCohortVecs for Vecs {
dateindex,
date_price,
exit,
&self.state,
self.state.as_mut().unwrap(),
)
}
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.inner
.safe_flush_stateful_vecs(height, exit, &mut self.state)
.safe_flush_stateful_vecs(height, exit, self.state.as_mut().unwrap())
}
#[allow(clippy::too_many_arguments)]
@@ -126,12 +132,12 @@ impl DynCohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.inner
.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
.compute_rest_part1(indexer, indexes, price, starting_indexes, exit)
}
fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -158,7 +164,7 @@ impl CohortVecs for Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
market: &market::Vecs,
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
@@ -166,11 +172,11 @@ impl CohortVecs for Vecs {
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.inner.compute_rest_part2(
indexer,
indexes,
fetched,
price,
starting_indexes,
market,
height_to_supply,
@@ -0,0 +1,54 @@
use brk_structs::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
#[derive(Debug)]
pub enum WithAddressDataSource<T> {
New(T),
FromLoadedAddressDataVec((LoadedAddressIndex, T)),
FromEmptyAddressDataVec((EmptyAddressIndex, T)),
}
impl<T> WithAddressDataSource<T> {
pub fn is_new(&self) -> bool {
matches!(self, Self::New(_))
}
pub fn is_from_emptyaddressdata(&self) -> bool {
matches!(self, Self::FromEmptyAddressDataVec(_))
}
pub fn deref_mut(&mut self) -> &mut T {
match self {
Self::New(v) => v,
Self::FromLoadedAddressDataVec((_, v)) => v,
Self::FromEmptyAddressDataVec((_, v)) => v,
}
}
}
impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
Self::FromLoadedAddressDataVec((i, v.into()))
}
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
Self::FromEmptyAddressDataVec((i, v.into()))
}
}
}
}
impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
Self::FromLoadedAddressDataVec((i, v.into()))
}
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
Self::FromEmptyAddressDataVec((i, v.into()))
}
}
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_core::{Dollars, Timestamp};
use brk_structs::{Dollars, Timestamp};
use super::SupplyState;
@@ -1,6 +1,7 @@
use std::path::Path;
use brk_core::{AddressData, Dollars, Height, Result, Sats};
use brk_error::Result;
use brk_structs::{Dollars, Height, LoadedAddressData, Sats};
use crate::SupplyState;
@@ -8,7 +9,7 @@ use super::CohortState;
#[derive(Clone)]
pub struct AddressCohortState {
pub address_count: usize,
pub address_count: u64,
pub inner: CohortState,
}
@@ -35,7 +36,7 @@ impl AddressCohortState {
#[allow(clippy::too_many_arguments)]
pub fn send(
&mut self,
addressdata: &mut AddressData,
addressdata: &mut LoadedAddressData,
value: Sats,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
@@ -47,14 +48,14 @@ impl AddressCohortState {
let prev_realized_price = compute_price.then(|| addressdata.realized_price());
let prev_supply_state = SupplyState {
utxos: addressdata.outputs_len as usize,
utxos: addressdata.outputs_len as u64,
value: addressdata.amount(),
};
addressdata.send(value, prev_price)?;
let supply_state = SupplyState {
utxos: addressdata.outputs_len as usize,
utxos: addressdata.outputs_len as u64,
value: addressdata.amount(),
};
@@ -72,19 +73,24 @@ impl AddressCohortState {
Ok(())
}
pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option<Dollars>) {
pub fn receive(
&mut self,
address_data: &mut LoadedAddressData,
value: Sats,
price: Option<Dollars>,
) {
let compute_price = price.is_some();
let prev_realized_price = compute_price.then(|| address_data.realized_price());
let prev_supply_state = SupplyState {
utxos: address_data.outputs_len as usize,
utxos: address_data.outputs_len as u64,
value: address_data.amount(),
};
address_data.receive(value, price);
let supply_state = SupplyState {
utxos: address_data.outputs_len as usize,
utxos: address_data.outputs_len as u64,
value: address_data.amount(),
};
@@ -96,7 +102,7 @@ impl AddressCohortState {
);
}
pub fn add(&mut self, addressdata: &AddressData) {
pub fn add(&mut self, addressdata: &LoadedAddressData) {
self.address_count += 1;
self.inner.increment_(
&addressdata.into(),
@@ -105,7 +111,7 @@ impl AddressCohortState {
);
}
pub fn subtract(&mut self, addressdata: &AddressData) {
pub fn subtract(&mut self, addressdata: &LoadedAddressData) {
self.address_count = self.address_count.checked_sub(1).unwrap();
self.inner.decrement_(
&addressdata.into(),
@@ -1,6 +1,7 @@
use std::{cmp::Ordering, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Result, Sats};
use brk_error::Result;
use brk_structs::{CheckedSub, Dollars, Height, Sats};
use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState};
@@ -52,12 +53,12 @@ impl CohortState {
pub fn increment(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.increment(supply_state, price);
self.price_to_amount.increment(price, supply_state);
}
if supply_state.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
let price = price.unwrap();
realized.increment(supply_state, price);
self.price_to_amount.increment(price, supply_state);
}
}
@@ -69,23 +70,23 @@ impl CohortState {
) {
self.supply += supply_state;
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
realized.increment_(realized_cap);
self.price_to_amount.increment(realized_price, supply_state);
}
if supply_state.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
realized.increment_(realized_cap);
self.price_to_amount.increment(realized_price, supply_state);
}
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply -= supply_state;
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.decrement(supply_state, price);
self.price_to_amount.decrement(price, supply_state);
}
if supply_state.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
let price = price.unwrap();
realized.decrement(supply_state, price);
self.price_to_amount.decrement(price, supply_state);
}
}
@@ -97,11 +98,11 @@ impl CohortState {
) {
self.supply -= supply_state;
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
realized.decrement_(realized_cap);
self.price_to_amount.decrement(realized_price, supply_state);
}
if supply_state.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
realized.decrement_(realized_cap);
self.price_to_amount.decrement(realized_price, supply_state);
}
}
@@ -123,21 +124,22 @@ impl CohortState {
) {
self.supply += supply_state;
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.receive(supply_state, price);
if supply_state.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
let price = price.unwrap();
realized.receive(supply_state, price);
if let Some((price, supply)) = price_to_amount_increment
&& supply.value.is_not_zero()
{
self.price_to_amount.increment(price, supply);
}
if let Some((price, supply)) = price_to_amount_decrement
&& supply.value.is_not_zero()
{
self.price_to_amount.decrement(price, supply);
}
if let Some((price, supply)) = price_to_amount_increment
&& supply.value.is_not_zero()
{
self.price_to_amount.increment(price, supply);
}
if let Some((price, supply)) = price_to_amount_decrement
&& supply.value.is_not_zero()
{
self.price_to_amount.decrement(price, supply);
}
}
}
@@ -223,17 +225,15 @@ impl CohortState {
let update_state =
|price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
match price.cmp(&current_price) {
Ordering::Equal => {
state.supply_even += sats;
}
Ordering::Less => {
state.supply_in_profit += sats;
if price > Dollars::ZERO && current_price > Dollars::ZERO {
let diff = current_price.checked_sub(price).unwrap();
if diff <= Dollars::ZERO {
dbg!(price, current_price, diff, sats);
panic!();
}
// Add back once in a while to verify, but generally not needed
// if diff <= Dollars::ZERO {
// dbg!(price, current_price, diff, sats);
// panic!();
// }
state.unrealized_profit += diff * sats;
}
}
@@ -241,13 +241,17 @@ impl CohortState {
state.supply_in_loss += sats;
if price > Dollars::ZERO && current_price > Dollars::ZERO {
let diff = price.checked_sub(current_price).unwrap();
if diff <= Dollars::ZERO {
dbg!(price, current_price, diff, sats);
panic!();
}
// Add back once in a while to verify, but generally not needed
// if diff <= Dollars::ZERO {
// dbg!(price, current_price, diff, sats);
// panic!();
// }
state.unrealized_loss += diff * sats;
}
}
Ordering::Equal => {
state.supply_even += sats;
}
}
};
@@ -1,6 +1,7 @@
use std::path::Path;
use brk_core::{Height, Result};
use brk_error::Result;
use brk_structs::Height;
use derive_deref::{Deref, DerefMut};
use super::CohortState;
@@ -1,14 +1,15 @@
use std::{
collections::BTreeMap,
fs::{self, File},
io::{BufReader, BufWriter},
fs,
io::{Cursor, Read},
path::{Path, PathBuf},
};
use bincode::{Decode, Encode, config, decode_from_std_read, encode_into_std_write};
use brk_core::{Dollars, Height, Result, Sats};
use brk_error::{Error, Result};
use brk_structs::{Dollars, Height, Sats};
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, IntoBytes};
use crate::states::SupplyState;
@@ -19,15 +20,15 @@ pub struct PriceToAmount {
state: State,
}
#[derive(Clone, Default, Debug, Deref, DerefMut, Serialize, Deserialize, Encode, Decode)]
struct State(BTreeMap<Dollars, Sats>);
impl PriceToAmount {
pub fn forced_import(path: &Path, name: &str) -> Self {
Self::import(path, name).unwrap_or_else(|_| Self {
pathbuf: Self::path_(path, name),
height: None,
state: State::default(),
Self::import(path, name).unwrap_or_else(|_| {
// dbg!(e);
Self {
pathbuf: Self::path_(path, name),
height: None,
state: State::default(),
}
})
}
@@ -35,10 +36,7 @@ impl PriceToAmount {
let path = Self::path_(path, name);
fs::create_dir_all(&path)?;
let config = config::standard();
let file = File::open(Self::path_state_(&path))?;
let mut reader = BufReader::new(file);
let state = decode_from_std_read(&mut reader, config)?;
let state = State::deserialize(&fs::read(Self::path_state_(&path))?)?;
Ok(Self {
height: Height::try_from(Self::path_height_(&path).as_path()).ok(),
@@ -68,10 +66,14 @@ impl PriceToAmount {
}
pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) {
let amount = self.state.get_mut(&price).unwrap();
*amount -= supply_state.value;
if *amount == Sats::ZERO {
self.state.remove(&price);
if let Some(amount) = self.state.get_mut(&price) {
*amount -= supply_state.value;
if *amount == Sats::ZERO {
self.state.remove(&price);
}
} else {
dbg!(&self.state, price, &self.pathbuf);
unreachable!();
}
}
@@ -86,13 +88,7 @@ impl PriceToAmount {
pub fn flush(&mut self, height: Height) -> Result<()> {
self.height = Some(height);
height.write(&self.path_height())?;
let config = config::standard();
let file = File::create(self.path_state()).inspect_err(|_| {
dbg!(self.path_state());
})?;
let mut writer = BufWriter::new(file);
encode_into_std_write(&self.state, &mut writer, config)?;
fs::write(self.path_state(), self.state.serialize())?;
Ok(())
}
@@ -118,3 +114,47 @@ impl PriceToAmount {
path.join("height")
}
}
#[derive(Clone, Default, Debug, Deref, DerefMut, Serialize, Deserialize)]
struct State(BTreeMap<Dollars, Sats>);
impl State {
fn serialize(&self) -> Vec<u8> {
let len = self.len();
let mut buffer = Vec::with_capacity(8 + len * 16);
buffer.extend(len.as_bytes());
self.iter().for_each(|(key, value)| {
buffer.extend(key.as_bytes());
buffer.extend(value.as_bytes());
});
buffer
}
fn deserialize(data: &[u8]) -> Result<Self> {
let mut cursor = Cursor::new(data);
let mut buffer = [0u8; 8];
cursor
.read_exact(&mut buffer)
.map_err(|_| Error::Str("Failed to read entry count"))?;
let entry_count = usize::read_from_bytes(&buffer)?;
let mut map = BTreeMap::new();
for _ in 0..entry_count {
cursor.read_exact(&mut buffer)?;
let key = Dollars::read_from_bytes(&buffer)?;
cursor.read_exact(&mut buffer)?;
let value = Sats::read_from_bytes(&buffer)?;
map.insert(key, value);
}
Ok(Self(map))
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
use std::cmp::Ordering;
use brk_core::{CheckedSub, Dollars};
use brk_structs::{CheckedSub, Dollars};
use super::SupplyState;
+5 -5
View File
@@ -1,12 +1,12 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_core::{AddressData, CheckedSub, Sats};
use brk_structs::{CheckedSub, LoadedAddressData, Sats};
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
pub struct SupplyState {
pub utxos: usize,
pub utxos: u64,
pub value: Sats,
}
@@ -40,10 +40,10 @@ impl SubAssign<&SupplyState> for SupplyState {
}
}
impl From<&AddressData> for SupplyState {
fn from(value: &AddressData) -> Self {
impl From<&LoadedAddressData> for SupplyState {
fn from(value: &LoadedAddressData) -> Self {
Self {
utxos: value.outputs_len as usize,
utxos: value.outputs_len as u64,
value: value.amount(),
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign};
use brk_core::{ByAmountRange, GroupedByType, OutputType, Sats};
use brk_structs::{ByAmountRange, GroupedByType, OutputType, Sats};
use super::SupplyState;
+1 -1
View File
@@ -1,4 +1,4 @@
use brk_core::{Dollars, Sats};
use brk_structs::{Dollars, Sats};
#[derive(Debug, Default, Clone)]
pub struct UnrealizedState {
-647
View File
@@ -1,647 +0,0 @@
use std::{path::Path, thread};
use brk_core::{
AddressData, ByAddressType, EmptyAddressData, Height, OutputType, P2AAddressIndex,
P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex,
P2WPKHAddressIndex, P2WSHAddressIndex, Result, TypeIndex, Version,
};
use brk_store::{AnyStore, Store};
use fjall::{PersistMode, TransactionalKeyspace};
use log::info;
use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Stores {
keyspace: TransactionalKeyspace,
pub p2aaddressindex_to_addressdata: Store<P2AAddressIndex, AddressData>,
pub p2aaddressindex_to_emptyaddressdata: Store<P2AAddressIndex, EmptyAddressData>,
pub p2pk33addressindex_to_addressdata: Store<P2PK33AddressIndex, AddressData>,
pub p2pk33addressindex_to_emptyaddressdata: Store<P2PK33AddressIndex, EmptyAddressData>,
pub p2pk65addressindex_to_addressdata: Store<P2PK65AddressIndex, AddressData>,
pub p2pk65addressindex_to_emptyaddressdata: Store<P2PK65AddressIndex, EmptyAddressData>,
pub p2pkhaddressindex_to_addressdata: Store<P2PKHAddressIndex, AddressData>,
pub p2pkhaddressindex_to_emptyaddressdata: Store<P2PKHAddressIndex, EmptyAddressData>,
pub p2shaddressindex_to_addressdata: Store<P2SHAddressIndex, AddressData>,
pub p2shaddressindex_to_emptyaddressdata: Store<P2SHAddressIndex, EmptyAddressData>,
pub p2traddressindex_to_addressdata: Store<P2TRAddressIndex, AddressData>,
pub p2traddressindex_to_emptyaddressdata: Store<P2TRAddressIndex, EmptyAddressData>,
pub p2wpkhaddressindex_to_addressdata: Store<P2WPKHAddressIndex, AddressData>,
pub p2wpkhaddressindex_to_emptyaddressdata: Store<P2WPKHAddressIndex, EmptyAddressData>,
pub p2wshaddressindex_to_addressdata: Store<P2WSHAddressIndex, AddressData>,
pub p2wshaddressindex_to_emptyaddressdata: Store<P2WSHAddressIndex, EmptyAddressData>,
}
impl Stores {
pub fn import(
path: &Path,
version: Version,
keyspace: &TransactionalKeyspace,
) -> color_eyre::Result<Self> {
let (
(p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata),
(p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata),
(p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata),
(p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata),
(p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata),
(p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata),
(p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata),
(p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata),
) = thread::scope(|scope| {
let p2a = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2aaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2aaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pk33 = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pk33addressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk33addressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pk65 = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pk65addressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk65addressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pkh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2sh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2shaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2shaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2tr = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2traddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2traddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2wpkh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2wsh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2wshaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wshaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
(
p2a.join().unwrap(),
p2pk33.join().unwrap(),
p2pk65.join().unwrap(),
p2pkh.join().unwrap(),
p2sh.join().unwrap(),
p2tr.join().unwrap(),
p2wpkh.join().unwrap(),
p2wsh.join().unwrap(),
)
});
Ok(Self {
keyspace: keyspace.clone(),
p2aaddressindex_to_addressdata,
p2aaddressindex_to_emptyaddressdata,
p2pk33addressindex_to_addressdata,
p2pk33addressindex_to_emptyaddressdata,
p2pk65addressindex_to_addressdata,
p2pk65addressindex_to_emptyaddressdata,
p2pkhaddressindex_to_addressdata,
p2pkhaddressindex_to_emptyaddressdata,
p2shaddressindex_to_addressdata,
p2shaddressindex_to_emptyaddressdata,
p2traddressindex_to_addressdata,
p2traddressindex_to_emptyaddressdata,
p2wpkhaddressindex_to_addressdata,
p2wpkhaddressindex_to_emptyaddressdata,
p2wshaddressindex_to_addressdata,
p2wshaddressindex_to_emptyaddressdata,
})
}
pub fn starting_height(&self) -> Height {
self.as_slice()
.into_iter()
.map(|store| store.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
pub fn reset(&mut self) -> Result<()> {
info!("Resetting stores...");
info!("> If it gets stuck here, stop the program and start it again");
self.as_mut_slice()
.into_iter()
.try_for_each(|store| store.reset())?;
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
}
pub fn get_addressdata(
&self,
address_type: OutputType,
type_index: TypeIndex,
) -> Result<Option<AddressData>> {
Ok(match address_type {
OutputType::P2A => self
.p2aaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK33 => self
.p2pk33addressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK65 => self
.p2pk65addressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PKH => self
.p2pkhaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2SH => self
.p2shaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2TR => self
.p2traddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WPKH => self
.p2wpkhaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WSH => self
.p2wshaddressindex_to_addressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
_ => unreachable!(),
})
}
pub fn get_emptyaddressdata(
&self,
address_type: OutputType,
type_index: TypeIndex,
) -> Result<Option<EmptyAddressData>> {
Ok(match address_type {
OutputType::P2A => self
.p2aaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK33 => self
.p2pk33addressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PK65 => self
.p2pk65addressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2PKH => self
.p2pkhaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2SH => self
.p2shaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2TR => self
.p2traddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WPKH => self
.p2wpkhaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
OutputType::P2WSH => self
.p2wshaddressindex_to_emptyaddressdata
.get(&type_index.into())?
.map(|c| c.into_owned()),
_ => unreachable!(),
})
}
pub fn commit(
&mut self,
height: Height,
addresstype_to_typeindex_to_addressdata: AddressTypeToTypeIndexTree<
WithAddressDataSource<AddressData>,
>,
addresstype_to_typeindex_to_emptyaddressdata: AddressTypeToTypeIndexTree<
WithAddressDataSource<EmptyAddressData>,
>,
) -> Result<()> {
let ByAddressType {
p2pk65,
p2pk33,
p2pkh,
p2sh,
p2wpkh,
p2wsh,
p2tr,
p2a,
} = addresstype_to_typeindex_to_addressdata.unwrap();
let ByAddressType {
p2pk65: empty_p2pk65,
p2pk33: empty_p2pk33,
p2pkh: empty_p2pkh,
p2sh: empty_p2sh,
p2wpkh: empty_p2wpkh,
p2wsh: empty_p2wsh,
p2tr: empty_p2tr,
p2a: empty_p2a,
} = addresstype_to_typeindex_to_emptyaddressdata.unwrap();
thread::scope(|s| {
s.spawn(|| {
self.p2aaddressindex_to_addressdata.commit_(
height,
empty_p2a
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2a.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2pk33addressindex_to_addressdata.commit_(
height,
empty_p2pk33
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2pk33.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2pk65addressindex_to_addressdata.commit_(
height,
empty_p2pk65
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2pk65.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2pkhaddressindex_to_addressdata.commit_(
height,
empty_p2pkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2pkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2shaddressindex_to_addressdata.commit_(
height,
empty_p2sh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2sh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2traddressindex_to_addressdata.commit_(
height,
empty_p2tr
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2tr.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2wpkhaddressindex_to_addressdata.commit_(
height,
empty_p2wpkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2wpkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
s.spawn(|| {
self.p2wshaddressindex_to_addressdata.commit_(
height,
empty_p2wsh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
.map(|(typeindex, _)| (*typeindex).into()),
p2wsh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
});
thread::scope(|scope| {
scope.spawn(|| {
self.p2aaddressindex_to_emptyaddressdata.commit_(
height,
p2a.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2a.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2pk33addressindex_to_emptyaddressdata.commit_(
height,
p2pk33
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2pk33.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2pk65addressindex_to_emptyaddressdata.commit_(
height,
p2pk65
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2pk65.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2pkhaddressindex_to_emptyaddressdata.commit_(
height,
p2pkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2pkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2shaddressindex_to_emptyaddressdata.commit_(
height,
p2sh.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2sh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2traddressindex_to_emptyaddressdata.commit_(
height,
p2tr.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2tr.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2wpkhaddressindex_to_emptyaddressdata.commit_(
height,
p2wpkh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2wpkh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
scope.spawn(|| {
self.p2wshaddressindex_to_emptyaddressdata.commit_(
height,
p2wsh
.iter()
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
.map(|(typeindex, _)| (*typeindex).into()),
empty_p2wsh.iter().map(|(typeindex, addressdata)| {
((*typeindex).into(), addressdata.deref().clone())
}),
)
});
});
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
}
pub fn rotate_memtables(&self) {
self.as_slice()
.into_iter()
.for_each(|store| store.rotate_memtable());
}
pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] {
[
&self.p2aaddressindex_to_addressdata,
&self.p2aaddressindex_to_emptyaddressdata,
&self.p2pk33addressindex_to_addressdata,
&self.p2pk33addressindex_to_emptyaddressdata,
&self.p2pk65addressindex_to_addressdata,
&self.p2pk65addressindex_to_emptyaddressdata,
&self.p2pkhaddressindex_to_addressdata,
&self.p2pkhaddressindex_to_emptyaddressdata,
&self.p2shaddressindex_to_addressdata,
&self.p2shaddressindex_to_emptyaddressdata,
&self.p2traddressindex_to_addressdata,
&self.p2traddressindex_to_emptyaddressdata,
&self.p2wpkhaddressindex_to_addressdata,
&self.p2wpkhaddressindex_to_emptyaddressdata,
&self.p2wshaddressindex_to_addressdata,
&self.p2wshaddressindex_to_emptyaddressdata,
]
}
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 16] {
[
&mut self.p2aaddressindex_to_addressdata,
&mut self.p2aaddressindex_to_emptyaddressdata,
&mut self.p2pk33addressindex_to_addressdata,
&mut self.p2pk33addressindex_to_emptyaddressdata,
&mut self.p2pk65addressindex_to_addressdata,
&mut self.p2pk65addressindex_to_emptyaddressdata,
&mut self.p2pkhaddressindex_to_addressdata,
&mut self.p2pkhaddressindex_to_emptyaddressdata,
&mut self.p2shaddressindex_to_addressdata,
&mut self.p2shaddressindex_to_emptyaddressdata,
&mut self.p2traddressindex_to_addressdata,
&mut self.p2traddressindex_to_emptyaddressdata,
&mut self.p2wpkhaddressindex_to_addressdata,
&mut self.p2wpkhaddressindex_to_emptyaddressdata,
&mut self.p2wshaddressindex_to_addressdata,
&mut self.p2wshaddressindex_to_emptyaddressdata,
]
}
}
+316
View File
@@ -0,0 +1,316 @@
use brk_error::Result;
use brk_structs::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, StoredIndex,
VecIterator, Version,
};
const DCA_AMOUNT: Dollars = Dollars::mint(100.0);
pub trait ComputeDCAStackViaLen {
fn compute_dca_stack_via_len(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
len: usize,
exit: &Exit,
) -> Result<()>;
fn compute_dca_stack_via_from(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
from: DateIndex,
exit: &Exit,
) -> Result<()>;
}
impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
fn compute_dca_stack_via_len(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + closes.version(),
)?;
let mut other_iter = closes.iter();
let mut prev = None;
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
let price = *closes.into_owned();
let i_usize = i.unwrap_to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1));
}
}
let mut stack = Sats::ZERO;
if price != Dollars::ZERO {
stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
if i_usize >= len {
let prev_price = *other_iter.unwrap_get_inner_(i_usize - len);
if prev_price != Dollars::ZERO {
stack = stack
.checked_sub(Sats::from(Bitcoin::from(DCA_AMOUNT / prev_price)))
.unwrap();
}
}
}
prev.replace(stack);
self.forced_push_at(i, stack, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
fn compute_dca_stack_via_from(
&mut self,
max_from: DateIndex,
closes: &impl AnyIterableVec<DateIndex, Close<Dollars>>,
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + closes.version(),
)?;
let mut prev = None;
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
let price = *closes.into_owned();
let i_usize = i.unwrap_to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1));
}
}
let mut stack = Sats::ZERO;
if price != Dollars::ZERO && i >= from {
stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
}
prev.replace(stack);
self.forced_push_at(i, stack, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeDCAAveragePriceViaLen {
fn compute_dca_avg_price_via_len(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
len: usize,
exit: &Exit,
) -> Result<()>;
fn compute_dca_avg_price_via_from(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
from: DateIndex,
exit: &Exit,
) -> Result<()>;
}
impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
fn compute_dca_avg_price_via_len(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ONE + self.inner_version() + stacks.version(),
)?;
let index = max_from.min(DateIndex::from(self.len()));
let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap();
stacks.iter_at(index).try_for_each(|(i, stack)| {
let stack = stack.into_owned();
let mut avg_price = Dollars::from(f64::NAN);
if i > first_price_date {
avg_price = DCA_AMOUNT
* len
.min(i.unwrap_to_usize() + 1)
.min(i.checked_sub(first_price_date).unwrap().unwrap_to_usize() + 1)
/ Bitcoin::from(stack);
}
self.forced_push_at(i, avg_price, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
fn compute_dca_avg_price_via_from(
&mut self,
max_from: DateIndex,
stacks: &impl AnyIterableVec<DateIndex, Sats>,
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + stacks.version(),
)?;
let index = max_from.min(DateIndex::from(self.len()));
let from_usize = from.unwrap_to_usize();
stacks.iter_at(index).try_for_each(|(i, stack)| {
let stack = stack.into_owned();
let mut avg_price = Dollars::from(f64::NAN);
if i >= from {
avg_price =
DCA_AMOUNT * (i.unwrap_to_usize() + 1 - from_usize) / Bitcoin::from(stack);
}
self.forced_push_at(i, avg_price, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeFromSats<I> {
fn compute_from_sats(
&mut self,
max_from: I,
sats: &impl AnyIterableVec<I, Sats>,
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeFromSats<I> for EagerVec<I, Bitcoin>
where
I: StoredIndex,
{
fn compute_from_sats(
&mut self,
max_from: I,
sats: &impl AnyIterableVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + sats.version(),
)?;
let index = max_from.min(I::from(self.len()));
sats.iter_at(index).try_for_each(|(i, sats)| {
let (i, v) = (i, Bitcoin::from(sats.into_owned()));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeFromBitcoin<I> {
fn compute_from_bitcoin(
&mut self,
max_from: I,
bitcoin: &impl AnyIterableVec<I, Bitcoin>,
price: &impl AnyIterableVec<I, Close<Dollars>>,
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeFromBitcoin<I> for EagerVec<I, Dollars>
where
I: StoredIndex,
{
fn compute_from_bitcoin(
&mut self,
max_from: I,
bitcoin: &impl AnyIterableVec<I, Bitcoin>,
price: &impl AnyIterableVec<I, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + bitcoin.version(),
)?;
let mut price_iter = price.iter();
let index = max_from.min(I::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| {
let dollars = price_iter.unwrap_get_inner(i);
let (i, v) = (i, *dollars * bitcoin.into_owned());
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
pub trait ComputeDrawdown<I> {
fn compute_drawdown(
&mut self,
max_from: I,
close: &impl AnyIterableVec<I, Close<Dollars>>,
ath: &impl AnyIterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeDrawdown<I> for EagerVec<I, StoredF32>
where
I: StoredIndex,
{
fn compute_drawdown(
&mut self,
max_from: I,
close: &impl AnyIterableVec<I, Close<Dollars>>,
ath: &impl AnyIterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + ath.version() + close.version(),
)?;
let index = max_from.min(I::from(self.len()));
let mut close_iter = close.iter();
ath.iter_at(index).try_for_each(|(i, ath)| {
let ath = ath.into_owned();
if ath == Dollars::ZERO {
self.forced_push_at(i, StoredF32::default(), exit)
} else {
let close = *close_iter.unwrap_get_inner(i);
let drawdown = StoredF32::from((*ath - *close) / *ath * -100.0);
self.forced_push_at(i, drawdown, exit)
}
})?;
self.safe_flush(exit)?;
Ok(())
}
}
@@ -1,108 +1,103 @@
use std::path::Path;
use brk_core::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
StoredUsize, TxIndex, TxVersion, Version, Weight,
};
use brk_exit::Exit;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
use brk_structs::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredBool,
StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Database, Exit, Format, PAGE_SIZE,
StoredIndex, VecIterator,
};
use crate::vecs::grouped::Source;
use super::{
Indexes, fetched,
grouped::{
ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
ComputedVecsFromTxindex, VecBuilderOptions,
},
indexes,
use crate::grouped::{
ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
ComputedVecsFromTxindex, Source, VecBuilderOptions,
};
use super::{Indexes, indexes, price};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
// pub txindex_to_is_v2: LazyVec<Txindex, bool>,
// pub txindex_to_is_v3: LazyVec<Txindex, bool>,
pub indexes_to_coinbase: ComputedValueVecsFromHeight,
pub indexes_to_emptyoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_emptyoutput_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_fee: ComputedValueVecsFromTxindex,
pub indexes_to_feerate: ComputedVecsFromTxindex<Feerate>,
/// Value == 0 when Coinbase
pub txindex_to_input_value: ComputedVecFrom3<
TxIndex,
Sats,
TxIndex,
InputIndex,
TxIndex,
StoredUsize,
InputIndex,
Sats,
>,
pub txindex_to_input_value:
ComputedVecFrom3<TxIndex, Sats, TxIndex, InputIndex, TxIndex, StoredU64, InputIndex, Sats>,
// pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredU64>,
pub txindex_to_output_value: ComputedVecFrom3<
TxIndex,
Sats,
TxIndex,
OutputIndex,
TxIndex,
StoredUsize,
StoredU64,
OutputIndex,
Sats,
>,
// pub indexes_to_output_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_p2a_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2ms_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2pk33_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2pk65_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2pkh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2sh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2tr_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2a_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2ms_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2pk33_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2pk65_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2pkh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2sh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2tr_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_subsidy: ComputedValueVecsFromHeight,
pub indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight,
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredUsize>,
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredU64>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredU64>,
pub inputindex_to_value:
ComputedVecFrom2<InputIndex, Sats, InputIndex, OutputIndex, OutputIndex, Sats>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_is_coinbase: ComputedVecFrom2<TxIndex, bool, TxIndex, Height, Height, TxIndex>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_vsize: ComputedVecFrom1<TxIndex, StoredUsize, TxIndex, Weight>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_is_coinbase:
ComputedVecFrom2<TxIndex, StoredBool, TxIndex, Height, Height, TxIndex>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_vsize: ComputedVecFrom1<TxIndex, StoredU64, TxIndex, Weight>,
pub txindex_to_weight:
ComputedVecFrom2<TxIndex, Weight, TxIndex, StoredU32, TxIndex, StoredU32>,
pub txindex_to_fee: ComputedVecFrom2<TxIndex, Sats, TxIndex, Sats, TxIndex, Sats>,
pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredUsize>,
pub indexes_to_exact_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredU64>,
pub indexes_to_exact_utxo_count: ComputedVecsFromHeight<StoredU64>,
}
impl Vecs {
pub fn forced_import(
path: &Path,
parent: &Path,
version: Version,
indexer: &Indexer,
indexes: &indexes::Vecs,
computation: Computation,
format: Format,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
price: Option<&price::Vecs>,
) -> Result<Self> {
let db = Database::open(&parent.join("transactions"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let compute_dollars = price.is_some();
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
computation,
path,
&db,
"value",
version + VERSION + Version::ZERO,
format,
@@ -129,7 +124,7 @@ impl Vecs {
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
computation,
path,
&db,
"weight",
version + VERSION + Version::ZERO,
format,
@@ -157,7 +152,7 @@ impl Vecs {
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
computation,
path,
&db,
"vsize",
version + VERSION + Version::ZERO,
format,
@@ -165,7 +160,7 @@ impl Vecs {
|index: TxIndex, iter| {
let index = index.unwrap_to_usize();
iter.next_at(index).map(|(_, weight)| {
StoredUsize::from(
StoredU64::from(
bitcoin::Weight::from(weight.into_owned()).to_vbytes_ceil() as usize
)
})
@@ -174,7 +169,7 @@ impl Vecs {
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
computation,
path,
&db,
"is_coinbase",
version + VERSION + Version::ZERO,
format,
@@ -190,15 +185,14 @@ impl Vecs {
.unwrap()
.1
.into_owned();
index == txindex
StoredBool::from(index == txindex)
})
},
)?;
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
computation,
path,
&db,
"input_value",
version + VERSION + Version::ZERO,
format,
@@ -219,7 +213,7 @@ impl Vecs {
.unwrap()
.1
.into_owned();
let range = first_index..first_index + count;
let range = first_index..first_index + count as usize;
range.into_iter().fold(Sats::ZERO, |total, inputindex| {
total
+ inputindex_to_value_iter
@@ -234,7 +228,7 @@ impl Vecs {
// let indexes_to_input_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// path,
// db,
// "input_value",
// true,
// version + VERSION + Version::ZERO,
@@ -248,7 +242,7 @@ impl Vecs {
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation,
path,
&db,
"output_value",
version + VERSION + Version::ZERO,
format,
@@ -269,7 +263,7 @@ impl Vecs {
.unwrap()
.1
.into_owned();
let range = first_index..first_index + count;
let range = first_index..first_index + count as usize;
range.into_iter().fold(Sats::ZERO, |total, outputindex| {
total
+ outputindex_to_value_iter
@@ -284,7 +278,7 @@ impl Vecs {
// let indexes_to_output_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// path,
// db,
// "output_value",
// true,
// version + VERSION + Version::ZERO,
@@ -297,8 +291,8 @@ impl Vecs {
// )?;
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
computation,
path,
Computation::Eager,
&db,
"fee",
version + VERSION + Version::ZERO,
format,
@@ -320,8 +314,8 @@ impl Vecs {
)?;
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
computation,
path,
Computation::Eager,
&db,
"feerate",
version + VERSION + Version::ZERO,
format,
@@ -341,7 +335,7 @@ impl Vecs {
Ok(Self {
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"tx_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -356,7 +350,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
path,
&db,
"input_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -371,7 +365,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
path,
&db,
"output_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -386,7 +380,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
path,
&db,
"tx_v1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -396,7 +390,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
path,
&db,
"tx_v2",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -406,7 +400,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
path,
&db,
"tx_v3",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -416,14 +410,14 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
path,
&db,
"fee",
indexes,
Source::Vec(txindex_to_fee.boxed_clone()),
version + VERSION + Version::ZERO,
computation,
format,
fetched,
price,
VecBuilderOptions::default()
.add_sum()
.add_cumulative()
@@ -432,7 +426,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
path,
&db,
"feerate",
Source::None,
version + VERSION + Version::ZERO,
@@ -445,7 +439,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
path,
&db,
"tx_vsize",
Source::None,
version + VERSION + Version::ZERO,
@@ -458,7 +452,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
path,
&db,
"tx_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -471,7 +465,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
path,
&db,
"subsidy",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -487,7 +481,7 @@ impl Vecs {
indexes,
)?,
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
path,
&db,
"coinbase",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -503,7 +497,7 @@ impl Vecs {
indexes,
)?,
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
path,
&db,
"unclaimed_rewards",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -514,7 +508,7 @@ impl Vecs {
indexes,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2a_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -529,7 +523,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2ms_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -544,7 +538,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2pk33_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -559,7 +553,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2pk65_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -574,7 +568,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2pkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -589,7 +583,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2sh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -604,7 +598,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2tr_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -619,7 +613,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2wpkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -634,7 +628,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"p2wsh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -649,7 +643,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"opreturn_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -664,7 +658,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"unknownoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -679,7 +673,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"emptyoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -694,7 +688,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
path,
&db,
"exact_utxo_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -713,6 +707,8 @@ impl Vecs {
txindex_to_feerate,
txindex_to_vsize,
txindex_to_weight,
db,
})
}
@@ -721,9 +717,22 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
fetched: Option<&fetched::Vecs>,
price: Option<&price::Vecs>,
exit: &Exit,
) -> color_eyre::Result<()> {
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, price, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
price: Option<&price::Vecs>,
exit: &Exit,
) -> Result<()> {
self.indexes_to_tx_count.compute_all(
indexer,
indexes,
@@ -735,7 +744,8 @@ impl Vecs {
&indexer.vecs.height_to_first_txindex,
&indexer.vecs.txindex_to_txid,
exit,
)
)?;
Ok(())
},
)?;
@@ -756,7 +766,7 @@ impl Vecs {
)?;
let compute_indexes_to_tx_vany =
|indexes_to_tx_vany: &mut ComputedVecsFromHeight<StoredUsize>, txversion| {
|indexes_to_tx_vany: &mut ComputedVecsFromHeight<StoredU64>, txversion| {
let mut txindex_to_txversion_iter = indexer.vecs.txindex_to_txversion.iter();
indexes_to_tx_vany.compute_all(
indexer,
@@ -773,7 +783,8 @@ impl Vecs {
v == txversion
},
exit,
)
)?;
Ok(())
},
)
};
@@ -867,7 +878,7 @@ impl Vecs {
starting_indexes,
exit,
Some(&self.txindex_to_fee),
fetched,
price,
)?;
self.indexes_to_feerate.compute_rest(
@@ -897,7 +908,7 @@ impl Vecs {
self.indexes_to_coinbase.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, indexer, _, starting_indexes, exit| {
@@ -914,23 +925,23 @@ impl Vecs {
.unwrap_to_usize();
let output_count = txindex_to_output_count_iter.unwrap_get_inner(txindex);
let mut sats = Sats::ZERO;
(first_outputindex..first_outputindex + *output_count).for_each(
|outputindex| {
(first_outputindex..first_outputindex + usize::from(output_count))
.for_each(|outputindex| {
sats += outputindex_to_value_iter
.unwrap_get_inner(OutputIndex::from(outputindex));
},
);
});
(height, sats)
},
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_subsidy.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -944,14 +955,15 @@ impl Vecs {
(height, coinbase.checked_sub(fees).unwrap())
},
exit,
)
)?;
Ok(())
},
)?;
self.indexes_to_unclaimed_rewards.compute_all(
indexer,
indexes,
fetched,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
@@ -965,7 +977,8 @@ impl Vecs {
(height, expected.checked_sub(subsidy).unwrap())
},
exit,
)
)?;
Ok(())
},
)?;
@@ -980,7 +993,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2aaddressindex,
&indexer.vecs.p2aaddressindex_to_p2abytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -995,7 +1009,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2msoutputindex,
&indexer.vecs.p2msoutputindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1010,7 +1025,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2pk33addressindex,
&indexer.vecs.p2pk33addressindex_to_p2pk33bytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1025,7 +1041,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2pk65addressindex,
&indexer.vecs.p2pk65addressindex_to_p2pk65bytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1040,7 +1057,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2pkhaddressindex,
&indexer.vecs.p2pkhaddressindex_to_p2pkhbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1055,7 +1073,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2shaddressindex,
&indexer.vecs.p2shaddressindex_to_p2shbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1070,7 +1089,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2traddressindex,
&indexer.vecs.p2traddressindex_to_p2trbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1085,7 +1105,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2wpkhaddressindex,
&indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1100,7 +1121,8 @@ impl Vecs {
&indexer.vecs.height_to_first_p2wshaddressindex,
&indexer.vecs.p2wshaddressindex_to_p2wshbytes,
exit,
)
)?;
Ok(())
},
)?;
@@ -1115,7 +1137,8 @@ impl Vecs {
&indexer.vecs.height_to_first_opreturnindex,
&indexer.vecs.opreturnindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1130,7 +1153,8 @@ impl Vecs {
&indexer.vecs.height_to_first_unknownoutputindex,
&indexer.vecs.unknownoutputindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1145,7 +1169,8 @@ impl Vecs {
&indexer.vecs.height_to_first_emptyoutputindex,
&indexer.vecs.emptyoutputindex_to_txindex,
exit,
)
)?;
Ok(())
},
)?;
@@ -1171,7 +1196,7 @@ impl Vecs {
|(h, output_count, ..)| {
let input_count = input_count_iter.unwrap_get_inner(h);
let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);
let block_count = usize::from(h + 1_usize);
let block_count = u64::from(h + 1_usize);
// -1 > genesis output is unspendable
let mut utxo_count =
*output_count - (*input_count - block_count) - *opreturn_count - 1;
@@ -1194,10 +1219,11 @@ impl Vecs {
utxo_count -= 1;
}
(h, StoredUsize::from(utxo_count))
(h, StoredU64::from(utxo_count))
},
exit,
)
)?;
Ok(())
},
)?;
@@ -1,14 +0,0 @@
use std::ops::{Add, AddAssign, Div};
use brk_vec::StoredType;
pub trait ComputedType
where
Self:
StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self> + AddAssign + Ord,
{
}
impl<T> ComputedType for T where
T: StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self> + AddAssign + Ord
{
}
-223
View File
@@ -1,223 +0,0 @@
use std::path::Path;
use brk_core::Version;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format};
pub mod blocks;
pub mod cointime;
pub mod constants;
pub mod fetched;
pub mod grouped;
pub mod indexes;
pub mod market;
pub mod mining;
pub mod stateful;
pub mod transactions;
pub use indexes::Indexes;
use log::info;
use crate::stores::Stores;
const VERSION: Version = Version::ONE;
#[derive(Clone)]
pub struct Vecs {
pub indexes: indexes::Vecs,
pub constants: constants::Vecs,
pub blocks: blocks::Vecs,
pub mining: mining::Vecs,
pub market: market::Vecs,
pub transactions: transactions::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
pub cointime: cointime::Vecs,
}
impl Vecs {
pub fn import(
path: &Path,
version: Version,
indexer: &Indexer,
fetch: bool,
computation: Computation,
format: Format,
) -> color_eyre::Result<Self> {
let indexes = indexes::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
indexer,
computation,
format,
)?;
let fetched = fetch.then(|| {
fetched::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)
.unwrap()
});
Ok(Self {
blocks: blocks::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
mining: mining::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
constants: constants::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
market: market::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
stateful: stateful::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
fetched.as_ref(),
)?,
transactions: transactions::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
indexer,
&indexes,
computation,
format,
fetched.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
path,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
fetched.as_ref(),
)?,
indexes,
fetched,
})
}
pub fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
fetcher: Option<&mut Fetcher>,
exit: &Exit,
stores: &mut Stores,
) -> color_eyre::Result<()> {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
self.constants
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing blocks...");
self.blocks
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing mining...");
self.mining
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(
indexer,
&self.indexes,
&starting_indexes,
fetcher.unwrap(),
exit,
)?;
}
info!("Computing transactions...");
self.transactions.compute(
indexer,
&self.indexes,
&starting_indexes,
self.fetched.as_ref(),
exit,
)?;
if let Some(fetched) = self.fetched.as_ref() {
info!("Computing market...");
self.market.compute(
indexer,
&self.indexes,
fetched,
&mut self.transactions,
&starting_indexes,
exit,
)?;
}
info!("Computing stateful...");
self.stateful.compute(
indexer,
&self.indexes,
&self.transactions,
self.fetched.as_ref(),
&self.market,
&mut starting_indexes,
exit,
stores,
)?;
self.cointime.compute(
indexer,
&self.indexes,
&starting_indexes,
self.fetched.as_ref(),
&self.transactions,
&self.stateful,
exit,
)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.constants.vecs(),
self.indexes.vecs(),
self.blocks.vecs(),
self.mining.vecs(),
self.market.vecs(),
self.transactions.vecs(),
self.stateful.vecs(),
self.cointime.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
@@ -1,47 +0,0 @@
use std::{collections::BTreeSet, mem};
use brk_core::TypeIndex;
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToTypeIndexSet(ByAddressType<BTreeSet<TypeIndex>>);
impl AddressTypeToTypeIndexSet {
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
self
}
fn merge_(own: &mut BTreeSet<TypeIndex>, other: &mut BTreeSet<TypeIndex>) {
if own.len() >= other.len() {
own.append(other);
} else {
other.append(own);
mem::swap(own, other);
}
}
}
impl Default for AddressTypeToTypeIndexSet {
fn default() -> Self {
Self(ByAddressType {
p2pk65: BTreeSet::default(),
p2pk33: BTreeSet::default(),
p2pkh: BTreeSet::default(),
p2sh: BTreeSet::default(),
p2wpkh: BTreeSet::default(),
p2wsh: BTreeSet::default(),
p2tr: BTreeSet::default(),
p2a: BTreeSet::default(),
})
}
}
@@ -1,21 +0,0 @@
use derive_deref::{Deref, DerefMut};
use super::ByAddressType;
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
impl<T> Default for AddressTypeToVec<T> {
fn default() -> Self {
Self(ByAddressType {
p2pk65: vec![],
p2pk33: vec![],
p2pkh: vec![],
p2sh: vec![],
p2wpkh: vec![],
p2wsh: vec![],
p2tr: vec![],
p2a: vec![],
})
}
}
File diff suppressed because it is too large Load Diff
@@ -1,62 +0,0 @@
use brk_core::{AddressData, EmptyAddressData};
#[derive(Debug)]
pub enum WithAddressDataSource<T> {
New(T),
FromAddressDataStore(T),
FromEmptyAddressDataStore(T),
}
impl<T> WithAddressDataSource<T> {
pub fn is_new(&self) -> bool {
matches!(self, Self::New(_))
}
pub fn is_from_addressdata(&self) -> bool {
matches!(self, Self::FromAddressDataStore(_))
}
pub fn is_from_emptyaddressdata(&self) -> bool {
matches!(self, Self::FromEmptyAddressDataStore(_))
}
pub fn deref(&self) -> &T {
match self {
Self::New(v) => v,
Self::FromAddressDataStore(v) => v,
Self::FromEmptyAddressDataStore(v) => v,
}
}
pub fn deref_mut(&mut self) -> &mut T {
match self {
Self::New(v) => v,
Self::FromAddressDataStore(v) => v,
Self::FromEmptyAddressDataStore(v) => v,
}
}
}
impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<AddressData> {
fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
WithAddressDataSource::FromEmptyAddressDataStore(v) => {
Self::FromEmptyAddressDataStore(v.into())
}
}
}
}
impl From<WithAddressDataSource<AddressData>> for WithAddressDataSource<EmptyAddressData> {
fn from(value: WithAddressDataSource<AddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
WithAddressDataSource::FromEmptyAddressDataStore(v) => {
Self::FromEmptyAddressDataStore(v.into())
}
}
}
}
-142
View File
@@ -1,142 +0,0 @@
use std::{
fmt::{self, Debug},
io, result, time,
};
use crate::Version;
pub type Result<T, E = Error> = result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
IO(io::Error),
SerdeJson(serde_json::Error),
Jiff(jiff::Error),
Fjall(fjall::Error),
SystemTimeError(time::SystemTimeError),
ZeroCopyError,
BincodeEncodeError(bincode::error::EncodeError),
BincodeDecodeError(bincode::error::DecodeError),
WrongEndian,
DifferentVersion { found: Version, expected: Version },
UnexpectedData,
MmapsVecIsTooSmall,
IndexTooHigh,
EmptyVec,
IndexTooLow,
ExpectFileToHaveIndex,
ExpectVecToHaveIndex,
FailedKeyTryIntoUsize,
UnsupportedUnflushedState,
RangeFromAfterTo(usize, usize),
DifferentCompressionMode,
WrongLength,
WrongAddressType,
UnindexableDate,
String(&'static str),
}
impl From<time::SystemTimeError> for Error {
fn from(value: time::SystemTimeError) -> Self {
Self::SystemTimeError(value)
}
}
impl From<io::Error> for Error {
fn from(value: io::Error) -> Self {
Self::IO(value)
}
}
impl From<jiff::Error> for Error {
fn from(value: jiff::Error) -> Self {
Self::Jiff(value)
}
}
impl From<fjall::Error> for Error {
fn from(value: fjall::Error) -> Self {
Self::Fjall(value)
}
}
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
}
}
impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
Self::ZeroCopyError
}
}
impl From<serde_json::Error> for Error {
fn from(error: serde_json::Error) -> Self {
Self::SerdeJson(error)
}
}
impl From<bincode::error::DecodeError> for Error {
fn from(error: bincode::error::DecodeError) -> Self {
Self::BincodeDecodeError(error)
}
}
impl From<bincode::error::EncodeError> for Error {
fn from(error: bincode::error::EncodeError) -> Self {
Self::BincodeEncodeError(error)
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::IO(error) => Debug::fmt(&error, f),
Error::SystemTimeError(error) => Debug::fmt(&error, f),
Error::SerdeJson(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::Fjall(error) => Debug::fmt(&error, f),
Error::BincodeDecodeError(error) => Debug::fmt(&error, f),
Error::BincodeEncodeError(error) => Debug::fmt(&error, f),
Error::ZeroCopyError => write!(f, "ZeroCopy error"),
Error::UnexpectedData => write!(f, "Unexpected data"),
Error::WrongEndian => write!(f, "Wrong endian"),
Error::DifferentVersion { found, expected } => {
write!(
f,
"Different version found: {found:?}, expected: {expected:?}"
)
}
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::IndexTooLow => write!(f, "Index too low"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"),
Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"),
Error::UnsupportedUnflushedState => {
write!(
f,
"Unsupported unflush state, please flush before using this function"
)
}
Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"),
Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(
f,
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
),
Error::String(s) => write!(f, "{s}"),
}
}
}
impl std::error::Error for Error {}
@@ -1,89 +0,0 @@
use std::ops::Add;
use byteview::ByteView;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct AddressIndex(u32);
impl AddressIndex {
pub const BYTES: usize = size_of::<Self>();
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(self.0 + 1)
}
}
impl From<u32> for AddressIndex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for AddressIndex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<AddressIndex> for u64 {
fn from(value: AddressIndex) -> Self {
value.0 as u64
}
}
impl From<usize> for AddressIndex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<AddressIndex> for usize {
fn from(value: AddressIndex) -> Self {
value.0 as usize
}
}
impl From<ByteView> for AddressIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<AddressIndex> for ByteView {
fn from(value: AddressIndex) -> Self {
Self::new(value.as_bytes())
}
}
impl Add<usize> for AddressIndex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u32)
}
}
impl Add<AddressIndex> for AddressIndex {
type Output = Self;
fn add(self, rhs: AddressIndex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
@@ -1,27 +0,0 @@
use byteview::ByteView;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
use super::{AddressIndex, Outputindex};
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes,
)]
#[repr(C)]
pub struct AddressIndexOutputIndex {
addressindex: AddressIndex,
_padding: u32,
outputindex: Outputindex,
}
impl From<ByteView> for AddressIndexOutputIndex {
fn from(value: ByteView) -> Self {
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<AddressIndexOutputIndex> for ByteView {
fn from(value: AddressIndexOutputIndex) -> Self {
Self::new(value.as_bytes())
}
}
@@ -1,48 +0,0 @@
use serde::Serialize;
use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
use super::OutputType;
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
TryFromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
#[repr(u8)]
pub enum AddressType {
P2PK65,
P2PK33,
P2PKH,
P2SH,
P2WPKH,
P2WSH,
P2TR,
P2A,
}
impl From<OutputType> for AddressType {
fn from(value: OutputType) -> Self {
match value {
OutputType::P2A => Self::P2A,
OutputType::P2PK33 => Self::P2PK33,
OutputType::P2PK65 => Self::P2PK65,
OutputType::P2PKH => Self::P2PKH,
OutputType::P2SH => Self::P2SH,
OutputType::P2TR => Self::P2TR,
OutputType::P2WPKH => Self::P2WPKH,
OutputType::P2WSH => Self::P2WSH,
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
}
}
}
@@ -1,44 +0,0 @@
use byteview::ByteView;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{AddressData, Sats};
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct EmptyAddressData {
pub transfered: Sats,
}
impl From<AddressData> for EmptyAddressData {
fn from(value: AddressData) -> Self {
Self::from(&value)
}
}
impl From<&AddressData> for EmptyAddressData {
fn from(value: &AddressData) -> Self {
if value.sent != value.received {
dbg!(&value);
panic!("Trying to convert not empty wallet to empty !");
}
Self {
transfered: value.sent,
}
}
}
impl From<ByteView> for EmptyAddressData {
fn from(value: ByteView) -> Self {
Self::read_from_bytes(&value).unwrap()
}
}
impl From<EmptyAddressData> for ByteView {
fn from(value: EmptyAddressData) -> Self {
Self::from(&value)
}
}
impl From<&EmptyAddressData> for ByteView {
fn from(value: &EmptyAddressData) -> Self {
Self::new(value.as_bytes())
}
}
-135
View File
@@ -1,135 +0,0 @@
use bitcoin::{ScriptBuf, opcodes::all::OP_PUSHBYTES_2};
use serde::Serialize;
use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
TryFromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
#[repr(u8)]
pub enum OutputType {
P2PK65,
P2PK33,
P2PKH,
P2MS,
P2SH,
OpReturn,
P2WPKH,
P2WSH,
P2TR,
P2A,
Empty = 254,
Unknown = 255,
}
impl OutputType {
pub fn is_spendable(&self) -> bool {
match self {
Self::P2PK65 => true,
Self::P2PK33 => true,
Self::P2PKH => true,
Self::P2MS => true,
Self::P2SH => true,
Self::OpReturn => false,
Self::P2WPKH => true,
Self::P2WSH => true,
Self::P2TR => true,
Self::P2A => true,
Self::Empty => true,
Self::Unknown => true,
}
}
pub fn is_address(&self) -> bool {
match self {
Self::P2PK65 => true,
Self::P2PK33 => true,
Self::P2PKH => true,
Self::P2MS => false,
Self::P2SH => true,
Self::OpReturn => false,
Self::P2WPKH => true,
Self::P2WSH => true,
Self::P2TR => true,
Self::P2A => true,
Self::Empty => false,
Self::Unknown => false,
}
}
pub fn is_not_address(&self) -> bool {
!self.is_address()
}
pub fn is_unspendable(&self) -> bool {
!self.is_spendable()
}
pub fn as_vec() -> Vec<Self> {
vec![
Self::P2PK65,
Self::P2PK33,
Self::P2PKH,
Self::P2MS,
Self::P2SH,
Self::OpReturn,
Self::P2WPKH,
Self::P2WSH,
Self::P2TR,
Self::P2A,
Self::Empty,
Self::Unknown,
]
}
}
impl From<&ScriptBuf> for OutputType {
fn from(script: &ScriptBuf) -> Self {
if script.is_p2pk() {
let bytes = script.as_bytes();
match bytes.len() {
67 => Self::P2PK65,
35 => Self::P2PK33,
_ => {
dbg!(bytes);
unreachable!()
}
}
} else if script.is_p2pkh() {
Self::P2PKH
} else if script.is_multisig() {
Self::P2MS
} else if script.is_p2sh() {
Self::P2SH
} else if script.is_op_return() {
Self::OpReturn
} else if script.is_p2wpkh() {
Self::P2WPKH
} else if script.is_p2wsh() {
Self::P2WSH
} else if script.is_p2tr() {
Self::P2TR
} else if script.witness_version() == Some(bitcoin::WitnessVersion::V1)
&& script.len() == 4
&& script.as_bytes()[1] == OP_PUSHBYTES_2.to_u8()
&& script.as_bytes()[2..4] == [78, 115]
{
Self::P2A
} else if script.is_empty() {
Self::Empty
} else {
Self::Unknown
}
}
}
-115
View File
@@ -1,115 +0,0 @@
use std::ops::{Add, AddAssign, Div};
use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Printable};
use super::{InputIndex, OutputIndex, TxIndex};
#[derive(
Debug,
Deref,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct StoredU64(u64);
impl StoredU64 {
pub const ZERO: Self = Self(0);
pub fn new(counter: u64) -> Self {
Self(counter)
}
}
impl From<u64> for StoredU64 {
fn from(value: u64) -> Self {
Self(value)
}
}
impl From<usize> for StoredU64 {
fn from(value: usize) -> Self {
Self(value as u64)
}
}
impl CheckedSub<StoredU64> for StoredU64 {
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self)
}
}
impl Div<usize> for StoredU64 {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
Self(self.0 / rhs as u64)
}
}
impl Add for StoredU64 {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl AddAssign for StoredU64 {
fn add_assign(&mut self, rhs: Self) {
*self = *self + rhs
}
}
impl From<f64> for StoredU64 {
fn from(value: f64) -> Self {
if value < 0.0 || value > u32::MAX as f64 {
panic!()
}
Self(value as u64)
}
}
impl From<StoredU64> for f64 {
fn from(value: StoredU64) -> Self {
value.0 as f64
}
}
impl From<TxIndex> for StoredU64 {
fn from(value: TxIndex) -> Self {
Self(*value as u64)
}
}
impl From<InputIndex> for StoredU64 {
fn from(value: InputIndex) -> Self {
Self(*value)
}
}
impl From<OutputIndex> for StoredU64 {
fn from(value: OutputIndex) -> Self {
Self(*value)
}
}
impl Printable for StoredU64 {
fn to_string() -> &'static str {
"u64"
}
fn to_possible_strings() -> &'static [&'static str] {
&["u64"]
}
}
-100
View File
@@ -1,100 +0,0 @@
use std::{
fs,
io::{self, Read},
iter::Sum,
ops::Add,
path::Path,
};
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Error, Result};
#[derive(
Default,
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
IntoBytes,
Immutable,
KnownLayout,
)]
pub struct Version(u64);
impl Version {
pub const ZERO: Self = Self(0);
pub const ONE: Self = Self(1);
pub const TWO: Self = Self(2);
pub const fn new(v: u64) -> Self {
Self(v)
}
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.as_bytes())
}
pub fn swap_bytes(self) -> Self {
Self(self.0.swap_bytes())
}
/// Ok(true) if existed and is same
/// Ok(false) if didn't exist
pub fn validate(&self, path: &Path) -> Result<bool> {
if let Ok(prev_version) = Version::try_from(path) {
if prev_version != *self {
if prev_version.swap_bytes() == *self {
return Err(Error::WrongEndian);
}
return Err(Error::DifferentVersion {
found: prev_version,
expected: *self,
});
}
Ok(true)
} else {
Ok(false)
}
}
}
impl From<Version> for u64 {
fn from(value: Version) -> u64 {
value.0
}
}
impl From<u64> for Version {
fn from(value: u64) -> Self {
Self(value)
}
}
impl TryFrom<&Path> for Version {
type Error = Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
let mut buf = [0; 8];
fs::read(value)?.as_slice().read_exact(&mut buf)?;
Ok(*(Self::ref_from_bytes(&buf)?))
}
}
impl Add<Version> for Version {
type Output = Self;
fn add(self, rhs: Version) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl Sum for Version {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
iter.fold(Self::ZERO, Add::add)
}
}
@@ -1,18 +0,0 @@
use std::ops::RangeInclusive;
pub trait FromCoarserIndex<T>
where
T: Ord + From<usize>,
{
fn min_from(coarser: T) -> usize;
fn max_from_(coarser: T) -> usize;
fn max_from(coarser: T, len: usize) -> usize {
Self::max_from_(coarser).min(len - 1)
}
fn inclusive_range_from(coarser: T, len: usize) -> RangeInclusive<usize>
where
T: Clone,
{
Self::min_from(coarser.clone())..=Self::max_from(coarser, len)
}
}
-5
View File
@@ -1,5 +0,0 @@
mod from_coarser;
mod printable;
pub use from_coarser::*;
pub use printable::*;
-4
View File
@@ -1,4 +0,0 @@
pub trait Printable {
fn to_string() -> &'static str;
fn to_possible_strings() -> &'static [&'static str];
}
-27
View File
@@ -1,27 +0,0 @@
use crate::{Error, Result};
#[allow(clippy::result_unit_err)]
pub fn copy_first_4bytes(slice: &[u8]) -> Result<[u8; 4]> {
let mut buf: [u8; 4] = [0; 4];
let buf_len = buf.len();
if slice.len() < buf_len {
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(buf)
}
#[allow(clippy::result_unit_err)]
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(buf)
}
-3
View File
@@ -1,3 +0,0 @@
pub trait CheckedSub<Rhs = Self>: Sized {
fn checked_sub(self, rhs: Rhs) -> Option<Self>;
}
-13
View File
@@ -1,13 +0,0 @@
mod bytes;
mod checked_sub;
mod paths;
mod pause;
mod rlimit;
mod serde;
pub use bytes::*;
pub use checked_sub::*;
pub use paths::*;
pub use pause::*;
pub use rlimit::*;
pub use serde::*;
-9
View File
@@ -1,9 +0,0 @@
use log::info;
pub fn pause() {
info!("Press enter to continue...");
let mut buffer = String::new();
std::io::stdin()
.read_line(&mut buffer)
.expect("Failed to read line");
}
-15
View File
@@ -1,15 +0,0 @@
use std::io;
use rlimit::{Resource, getrlimit};
pub fn setrlimit() -> io::Result<()> {
let no_file_limit = getrlimit(Resource::NOFILE)?;
rlimit::setrlimit(
Resource::NOFILE,
no_file_limit.0.max(10_000),
no_file_limit.1,
)?;
Ok(())
}

Some files were not shown because too many files have changed in this diff Show More