Compare commits
373 Commits
kibo-v0.5.0
...
v0.0.76
| Author | SHA1 | Date | |
|---|---|---|---|
| a66f4ad4bd | |||
| 1dd687dab7 | |||
| 50ff6e2745 | |||
| 811dec713b | |||
| 617d6f4bd7 | |||
| 57cd2d6252 | |||
| ec64f8d048 | |||
| ed288a9dba | |||
| 27da0a4102 | |||
| 3c01ba1a76 | |||
| 252c8833ae | |||
| f45fb6efe6 | |||
| 8cc1f8d691 | |||
| bff22b5182 | |||
| d31d47eb32 | |||
| 5fe984c39d | |||
| 7f07b0daa7 | |||
| 6d35c26b3f | |||
| be4e693a27 | |||
| 5810276156 | |||
| d10ac3f87b | |||
| 9810bc09e9 | |||
| a0a13eb2a8 | |||
| 6e996797b8 | |||
| 663092b501 | |||
| 8ea13544de | |||
| e73daa6214 | |||
| cf92c60a01 | |||
| b7f51b03bc | |||
| 903e69ff77 | |||
| c4167ddaad | |||
| 50bfdb0d68 | |||
| a6cb09ff1c | |||
| e4c9f23476 | |||
| 44e5415d43 | |||
| 1c653693ed | |||
| 39c470ad7a | |||
| 1103e538a5 | |||
| c0cd4cba6f | |||
| b91120e8d4 | |||
| 005774a4c2 | |||
| 16bbfebfba | |||
| 15505cd82d | |||
| 016d80e002 | |||
| 0f3c267a48 | |||
| 589bb02411 | |||
| c0f4ece17b | |||
| c3ae3cb768 | |||
| c9e0f9d985 | |||
| e3431c2fa3 | |||
| 5979b9771e | |||
| aa61832fb2 | |||
| 2ac6e982b1 | |||
| 3204ddcf07 | |||
| c87b1c133c | |||
| 9b275ecdae | |||
| d6fd7de361 | |||
| 49d66a133e | |||
| c559f26d0e | |||
| bbe9f1bad2 | |||
| 7e1fb6472d | |||
| 0ff8d20573 | |||
| 9c1f9448dc | |||
| 43a6081dd6 | |||
| 985e961876 | |||
| 098f6de047 | |||
| 1b0f90fd68 | |||
| 12252f407b | |||
| 3b6e3f47ab | |||
| 6a9ac9b025 | |||
| ae6aa4088b | |||
| c08f431180 | |||
| 123c1f56e9 | |||
| 35ac65a864 | |||
| e9f362cc87 | |||
| 65685c23e1 | |||
| 2f74748cea | |||
| f477bd66f3 | |||
| d7d77ae8f0 | |||
| 31110a740d | |||
| b64d8b1d7f | |||
| c46006aacc | |||
| 92f81b1493 | |||
| 70213cfc8f | |||
| 8a82bf5c50 | |||
| 37405384a2 | |||
| 54ea6cc53b | |||
| 339c00d815 | |||
| ea6b4dcde2 | |||
| 2b84623d1e | |||
| c8b3afa56b | |||
| 1348f3c24c | |||
| 62208ce3e1 | |||
| 813b2481de | |||
| 27b924ba61 | |||
| b40170b8ce | |||
| 8bfa9d2734 | |||
| c7cf76d4a8 | |||
| dfd2969b3e | |||
| 0e1866fe1d | |||
| b9ae46b913 | |||
| 06e7284055 | |||
| 93289e8fca | |||
| 130d5057d4 | |||
| be492d5084 | |||
| e0bf1d736f | |||
| 5a6b71cbeb | |||
| e6934cd5e2 | |||
| b5aada0792 | |||
| 165ea83ac3 | |||
| 440a82dee4 | |||
| 9c2d3e5e26 | |||
| 6fb6abcbe5 | |||
| dc449dafd1 | |||
| ecdaeebbfb | |||
| fa958b59bd | |||
| fb3d8521cd | |||
| 608c401cf3 | |||
| 1c3da90a24 | |||
| 34567f3375 | |||
| 51bcbeb48f | |||
| cc0f9c42df | |||
| a11bf5523b | |||
| 1921c3d901 | |||
| d568469e8b | |||
| 20d5c7e8d5 | |||
| 9f289ed9de | |||
| 93ee5e480b | |||
| 98a312701f | |||
| cbcf603b63 | |||
| f976f672cf | |||
| cfc3081e8a | |||
| 99818924ee | |||
| 9bbf3a027f | |||
| 93e01902e3 | |||
| 34919aba05 | |||
| a8ee4cf57f | |||
| b39548b4c6 | |||
| 4217c22ff6 | |||
| 4ab10670c9 | |||
| 2883f88de6 | |||
| e002a61a19 | |||
| 5893376279 | |||
| 411c5e4c4d | |||
| c2a77072d2 | |||
| c8a25934a6 | |||
| 7b38355cd4 | |||
| ddc54e0b98 | |||
| 8a7003782b | |||
| 8e6464dacb | |||
| 92b1dc0afb | |||
| 7562f51e07 | |||
| 09bba99e68 | |||
| 9d674cd49b | |||
| 88a0c9ea03 | |||
| 5014e0ce3e | |||
| b7a1ee9ebc | |||
| 292ceddd66 | |||
| 4b52b80000 | |||
| 9f20664c6e | |||
| 851a6aac0e | |||
| 1f1e73c47a | |||
| 112f61ca18 | |||
| 96eeacbe2b | |||
| 3f62da879c | |||
| aa30feb875 | |||
| 9ba3c2b7c5 | |||
| 320c708e10 | |||
| efa7294f59 | |||
| ae0e092935 | |||
| c77aecbfce | |||
| 700352ec45 | |||
| 664b125ce2 | |||
| 5f4b1c9e32 | |||
| d11d3f19bd | |||
| f34f4f2738 | |||
| 15db7c2310 | |||
| f9257ed04d | |||
| 15e6ef8488 | |||
| 9ae0a57f22 | |||
| 1e38c21f8e | |||
| bdc3c19163 | |||
| d55478da54 | |||
| 82bcc55645 | |||
| 07618ebe43 | |||
| 1492834d1e | |||
| 5ab6197356 | |||
| 0a789fe551 | |||
| caa8ff23ed | |||
| ee30d1d36d | |||
| 0d9415db9d | |||
| 8020e1126f | |||
| 3439422057 | |||
| 68d2bf736f | |||
| d78c39fd8c | |||
| b1dcad86b4 | |||
| 9b6124074d | |||
| 02cbaa1e80 | |||
| a12f1321c7 | |||
| 8b67f592ac | |||
| 319d17b337 | |||
| 476eaa85da | |||
| d26099855c | |||
| e47456da17 | |||
| a464d5d0b6 | |||
| 1cfb7b5615 | |||
| ac7c2f3d03 | |||
| 638d9e6e01 | |||
| 8b9df2a396 | |||
| d7fe911bde | |||
| 0acc3d511b | |||
| 4cf465f419 | |||
| b686d317a9 | |||
| dcef541852 | |||
| abdd733f11 | |||
| 942431e882 | |||
| 1c75ea046c | |||
| f32b6daa51 | |||
| 3736d6ba5e | |||
| 9788b01f35 | |||
| 9aec991da6 | |||
| 910701ce04 | |||
| 34b462d511 | |||
| 139e93b2f0 | |||
| 0dd7e9359e | |||
| 41cf0225e3 | |||
| 962254e511 | |||
| a7f2b24bac | |||
| 1323d988af | |||
| 7c49e5c749 | |||
| cd69ec4fa3 | |||
| 4c7e9fbee2 | |||
| 1639df5616 | |||
| 810cdbd790 | |||
| 0d4f4aec4e | |||
| 6b1863d3b4 | |||
| 27f5a3b16b | |||
| 876cd8291b | |||
| d0c46e4ef3 | |||
| feb8898ebf | |||
| 4fef8c5cfd | |||
| 7d56d8e35b | |||
| 5f1a3a9c8f | |||
| 0767b3156d | |||
| 9f16379b41 | |||
| be632aaf37 | |||
| 118c87faf7 | |||
| ec1e53d566 | |||
| 6a17ee414a | |||
| 6700686e4b | |||
| e8c34dd59b | |||
| 4c2da31bb3 | |||
| c0144b99bf | |||
| a0c32fc146 | |||
| a07b641adb | |||
| 0bb869fb33 | |||
| 72389e0129 | |||
| f49529fa70 | |||
| afcc34b5cc | |||
| 655b99cac8 | |||
| cc5091e28c | |||
| 1c72362c6b | |||
| 50ad5f681b | |||
| 50bf670931 | |||
| 7a8896864f | |||
| 51fbf148d9 | |||
| a9929438cd | |||
| 5a94b6b56c | |||
| 52cfbf60d4 | |||
| 29c10f8854 | |||
| ad761e388d | |||
| 07493ab0a6 | |||
| 7441011ae7 | |||
| d0818f456d | |||
| 06ea07a021 | |||
| 36d97ad5ca | |||
| a995eb2929 | |||
| c459a3033d | |||
| b4fbcf6bee | |||
| b9e679a514 | |||
| 64d73b93e4 | |||
| db70b05088 | |||
| 9428beeae5 | |||
| f9f7172702 | |||
| f1851b304c | |||
| d2ca6f1d46 | |||
| b27297cdc6 | |||
| 0d0edd7917 | |||
| fc6f12fb22 | |||
| d24096374f | |||
| 4f1d04009a | |||
| 79e9fde937 | |||
| 0ebaf6a171 | |||
| be2012f28d | |||
| ceefc8ffc6 | |||
| 0453b6903a | |||
| 691952249b | |||
| 0ceae2852e | |||
| 6d7ff38cf2 | |||
| 1b93ccf608 | |||
| 5b1ca3711a | |||
| 877f9299e1 | |||
| 677aca7a03 | |||
| 66b31a62d0 | |||
| 34923638c5 | |||
| bb61b3dc22 | |||
| 01ecae8979 | |||
| 53175c9ed7 | |||
| bc7a76755b | |||
| 92758f3e4e | |||
| b09767c526 | |||
| 2f93fd7c36 | |||
| 8acbcc548c | |||
| 19cf34f9d4 | |||
| 8c3f519016 | |||
| e63b42278c | |||
| 66ecd2fcf8 | |||
| f0d86f2392 | |||
| 5e39510f21 | |||
| 2cb4d65f3d | |||
| 15f2e05192 | |||
| a122333aaa | |||
| 06b2186bf9 | |||
| ed10dccfe2 | |||
| a1006dddb5 | |||
| 443a32dc81 | |||
| b034b4fe2f | |||
| 27b270148b | |||
| 269c64e4ed | |||
| eaf76e27f5 | |||
| 385b881068 | |||
| cf26696d12 | |||
| cb7ea40e7c | |||
| 5aaa55197e | |||
| d86d614520 | |||
| 138ca80c10 | |||
| d11a1622f8 | |||
| 42c996e16e | |||
| 1e37d75e49 | |||
| ad34d9d402 | |||
| 8c610f8a83 | |||
| f7f3e3cc03 | |||
| d68c6f9f2e | |||
| 90a5c4fbf8 | |||
| 042be6e229 | |||
| 4923c2e204 | |||
| b94d94e116 | |||
| d629ae8fbb | |||
| 1296a2e9ec | |||
| 009d02fa68 | |||
| 4cc57e9c91 | |||
| d373c6398e | |||
| 82746a0669 | |||
| 1212c3627b | |||
| 813f16ccee | |||
| 1c3cb91ecd | |||
| 5b1735db2b | |||
| bf31ee5fd6 | |||
| 1380b42c1d | |||
| dea853d840 | |||
| d72bf0739a | |||
| 481f5c0a97 | |||
| 2b017ac6b5 | |||
| 8a733ee337 | |||
| 9dd87a48a6 | |||
| 8fabbde13b | |||
| e0a378cb81 | |||
| 0b3329ca35 | |||
| 50c77b51db | |||
| c883ed19d6 | |||
| 795791219e | |||
| f6f4660cd2 | |||
| 9576f6e91e |
@@ -0,0 +1,291 @@
|
||||
# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/
|
||||
#
|
||||
# Copyright 2022-2024, axodotdev
|
||||
# SPDX-License-Identifier: MIT or Apache-2.0
|
||||
#
|
||||
# CI that:
|
||||
#
|
||||
# * checks for a Git Tag that looks like a release
|
||||
# * builds artifacts with dist (archives, installers, hashes)
|
||||
# * uploads those artifacts to temporary workflow zip
|
||||
# * on success, uploads the artifacts to a GitHub Release
|
||||
#
|
||||
# Note that the GitHub Release will be created with a generated
|
||||
# title/body based on your changelogs.
|
||||
|
||||
name: Release
|
||||
permissions:
|
||||
"contents": "write"
|
||||
|
||||
# This task will run whenever you push a git tag that looks like a version
|
||||
# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
|
||||
# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
|
||||
# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
|
||||
# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
|
||||
#
|
||||
# If PACKAGE_NAME is specified, then the announcement will be for that
|
||||
# package (erroring out if it doesn't have the given version or isn't dist-able).
|
||||
#
|
||||
# If PACKAGE_NAME isn't specified, then the announcement will be for all
|
||||
# (dist-able) packages in the workspace with that version (this mode is
|
||||
# intended for workspaces with only one dist-able package, or with all dist-able
|
||||
# packages versioned/released in lockstep).
|
||||
#
|
||||
# If you push multiple tags at once, separate instances of this workflow will
|
||||
# spin up, creating an independent announcement for each one. However, GitHub
|
||||
# will hard limit this to 3 tags per commit, as it will assume more tags is a
|
||||
# mistake.
|
||||
#
|
||||
# If there's a prerelease-style suffix to the version, then the release(s)
|
||||
# will be marked as a prerelease.
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
tags:
|
||||
- '**[0-9]+.[0-9]+.[0-9]+*'
|
||||
|
||||
jobs:
|
||||
# Run 'dist plan' (or host) to determine what tasks we need to do
|
||||
plan:
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
val: ${{ steps.plan.outputs.manifest }}
|
||||
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
|
||||
tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
|
||||
publishing: ${{ !github.event.pull_request }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install dist
|
||||
# we specify bash to get pipefail; it guards against the `curl` command
|
||||
# failing. otherwise `sh` won't catch that `curl` returned non-0
|
||||
shell: bash
|
||||
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh"
|
||||
- name: Cache dist
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/dist
|
||||
# sure would be cool if github gave us proper conditionals...
|
||||
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
|
||||
# functionality based on whether this is a pull_request, and whether it's from a fork.
|
||||
# (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
|
||||
# but also really annoying to build CI around when it needs secrets to work right.)
|
||||
- id: plan
|
||||
run: |
|
||||
dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
cat plan-dist-manifest.json
|
||||
echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
- name: "Upload dist-manifest.json"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-plan-dist-manifest
|
||||
path: plan-dist-manifest.json
|
||||
|
||||
# Build and packages all the platform-specific things
|
||||
build-local-artifacts:
|
||||
name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
|
||||
# Let the initial task tell us to not run (currently very blunt)
|
||||
needs:
|
||||
- plan
|
||||
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# Target platforms/runners are computed by dist in create-release.
|
||||
# Each member of the matrix has the following arguments:
|
||||
#
|
||||
# - runner: the github runner
|
||||
# - dist-args: cli flags to pass to dist
|
||||
# - install-dist: expression to run to install dist on the runner
|
||||
#
|
||||
# Typically there will be:
|
||||
# - 1 "global" task that builds universal installers
|
||||
# - N "local" tasks that build each platform's binaries and platform-specific installers
|
||||
matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.container && matrix.container.image || null }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
|
||||
steps:
|
||||
- name: enable windows longpaths
|
||||
run: |
|
||||
git config --global core.longpaths true
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install Rust non-interactively if not already installed
|
||||
if: ${{ matrix.container }}
|
||||
run: |
|
||||
if ! command -v cargo > /dev/null 2>&1; then
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
fi
|
||||
- name: Install dist
|
||||
run: ${{ matrix.install_dist.run }}
|
||||
# Get the dist-manifest
|
||||
- name: Fetch local artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
${{ matrix.packages_install }}
|
||||
- name: Build artifacts
|
||||
run: |
|
||||
# Actually do builds and make zips and whatnot
|
||||
dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
- id: cargo-dist
|
||||
name: Post-build
|
||||
# We force bash here just because github makes it really hard to get values up
|
||||
# to "real" actions without writing to env-vars, and writing to env-vars has
|
||||
# inconsistent syntax between shell and powershell.
|
||||
shell: bash
|
||||
run: |
|
||||
# Parse out what we just built and upload it to scratch storage
|
||||
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
|
||||
dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
|
||||
- name: "Upload artifacts"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-build-local-${{ join(matrix.targets, '_') }}
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
|
||||
# Build and package all the platform-agnostic(ish) things
|
||||
build-global-artifacts:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
runs-on: "ubuntu-latest"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/
|
||||
- run: chmod +x ~/.cargo/bin/dist
|
||||
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
|
||||
- name: Fetch local artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- id: cargo-dist
|
||||
shell: bash
|
||||
run: |
|
||||
dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
|
||||
# Parse out what we just built and upload it to scratch storage
|
||||
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
|
||||
jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
|
||||
- name: "Upload artifacts"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-build-global
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
# Determines if we should publish/announce
|
||||
host:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
- build-global-artifacts
|
||||
# Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
val: ${{ steps.host.outputs.manifest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/
|
||||
- run: chmod +x ~/.cargo/bin/dist
|
||||
# Fetch artifacts from scratch-storage
|
||||
- name: Fetch artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- id: host
|
||||
shell: bash
|
||||
run: |
|
||||
dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
|
||||
echo "artifacts uploaded and released successfully"
|
||||
cat dist-manifest.json
|
||||
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
- name: "Upload dist-manifest.json"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
# Overwrite the previous copy
|
||||
name: artifacts-dist-manifest
|
||||
path: dist-manifest.json
|
||||
# Create a GitHub Release while uploading all files to it
|
||||
- name: "Download GitHub Artifacts"
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: artifacts
|
||||
merge-multiple: true
|
||||
- name: Cleanup
|
||||
run: |
|
||||
# Remove the granular manifests
|
||||
rm -f artifacts/*-dist-manifest.json
|
||||
- name: Create GitHub Release
|
||||
env:
|
||||
PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
|
||||
ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}"
|
||||
ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}"
|
||||
RELEASE_COMMIT: "${{ github.sha }}"
|
||||
run: |
|
||||
# Write and read notes from a file to avoid quoting breaking things
|
||||
echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt
|
||||
|
||||
gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
|
||||
|
||||
announce:
|
||||
needs:
|
||||
- plan
|
||||
- host
|
||||
# use "always() && ..." to allow us to wait for all publish jobs while
|
||||
# still allowing individual publish jobs to skip themselves (for prereleases).
|
||||
# "host" however must run to completion, no skipping allowed!
|
||||
if: ${{ always() && needs.host.result == 'success' }}
|
||||
runs-on: "ubuntu-latest"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
@@ -1,59 +1,20 @@
|
||||
# Mac OS
|
||||
.DS_Store
|
||||
|
||||
# To do
|
||||
/charts
|
||||
TODO.md
|
||||
|
||||
# Builds
|
||||
dist
|
||||
target
|
||||
|
||||
# I/O
|
||||
in
|
||||
out
|
||||
.log
|
||||
/datasets
|
||||
/datasets2
|
||||
/price
|
||||
*..*
|
||||
/txout_*
|
||||
/db
|
||||
|
||||
# Sync
|
||||
.stfolder
|
||||
dist
|
||||
vecid-to-indexes.js
|
||||
|
||||
# Copies
|
||||
*\ copy*
|
||||
|
||||
# Ignored
|
||||
ignore
|
||||
|
||||
# Scripts
|
||||
/start-node.sh
|
||||
_*
|
||||
|
||||
# Editors
|
||||
.vscode
|
||||
.zed
|
||||
|
||||
# Configs
|
||||
config.toml
|
||||
|
||||
# Flamegraph
|
||||
flamegraph/
|
||||
flamegraph.svg
|
||||
|
||||
# Benchmarks
|
||||
benches
|
||||
|
||||
# Snapshots
|
||||
snapshots*/
|
||||
|
||||
# Docker
|
||||
docker/kibo
|
||||
|
||||
# Types
|
||||
website/scripts/types/paths.d.ts
|
||||
|
||||
# Misc
|
||||
OPENSATS.md
|
||||
# Logs
|
||||
.log
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
# Changelog
|
||||
|
||||
<!--
|
||||
## v. 0.X.Y | WIP
|
||||

|
||||
# v0.X.Y | WIP
|
||||

|
||||
-->
|
||||
|
||||
## v. 0.5.0 | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
# v0.X.0 | WIP | A new beginning
|
||||
|
||||

|
||||

|
||||
|
||||
Full rewrite
|
||||
|
||||
# [kibo-v0.5.0](https://github.com/bitcoinresearchkit/brk/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
|
||||

|
||||
|
||||
## Datasets
|
||||
|
||||
@@ -72,15 +76,15 @@
|
||||
|
||||
- Moved back to this repo
|
||||
|
||||
## v. 0.4.0 | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
# [kibo-v0.4.0](https://github.com/bitcoinresearchkit/brk/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
|
||||

|
||||

|
||||
|
||||
### Brand
|
||||
## Brand
|
||||
|
||||
- **Satonomics** is now **kibō** 🎉
|
||||
- **Satonomics** is now **kibo** 🎉
|
||||
|
||||
### Website
|
||||
## Website
|
||||
|
||||
- Complete redesign of the website
|
||||
- Rewrote the whole application and removed `node`/`npm`/`pnpm` dependencies in favor for pure `HTML`/`CSS`/`Javascript`
|
||||
@@ -88,7 +92,7 @@
|
||||
- Added Trading View attribution link to the settings frame and file in the lightweight charts folder
|
||||
- Many other changes
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
|
||||
- Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage
|
||||
@@ -103,17 +107,17 @@
|
||||
- Various first run fixes
|
||||
- Added to `-h` which arguments are saved, which is all of them at the time of writing
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Updated the code to support compressed binaries
|
||||
- Added serving of the website
|
||||
- Improved `Cache-Control` behavior
|
||||
|
||||
## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
|
||||
# [satonomics-v0.3.0](https://github.com/bitcoinresearchkit/brk/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
|
||||
|
||||

|
||||

|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Global
|
||||
- Improved self-hosting by:
|
||||
@@ -156,7 +160,7 @@
|
||||
- Price
|
||||
- Improved error message when price cannot be found
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added chart scroll button for nice animations à la Wicked
|
||||
@@ -182,17 +186,17 @@
|
||||
- Settings
|
||||
- Removed the horizontal scroll bar which was unintended
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Run file
|
||||
- Only run with a watcher if `cargo watch` is available
|
||||
- Removed id_to_path file in favor for only `paths.d.ts` in `app/src/types`
|
||||
|
||||
## v. 0.2.0 | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
|
||||
# [satonomics-v0.2.0](https://github.com/bitcoinresearchkit/brk/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
|
||||
|
||||

|
||||

|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added the height version of all datasets and many optimizations to make them usable but only available on desktop and tablets for now
|
||||
@@ -220,24 +224,24 @@
|
||||
- Hopefully made scrollbars a little more subtle on WIndows and Linux, can't test
|
||||
- Generale style updates
|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Fixed ulimit only being run in Mac OS instead of whenever the program is detected
|
||||
|
||||
## v. 0.1.1 | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
|
||||
# [satonomics-v0.1.1](https://github.com/bitcoinresearchkit/brk/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
|
||||
|
||||

|
||||

|
||||
|
||||
### Parser
|
||||
## Parser
|
||||
|
||||
- Fixed overflow in `Price` struct which caused many Realized Caps and Realized Prices to have completely bogus data
|
||||
- Fixed Realized Cap computation which was using rounded prices instead normal ones
|
||||
|
||||
### Server
|
||||
## Server
|
||||
|
||||
- Added the chunk, date and time of the request to the terminal logs
|
||||
|
||||
### App
|
||||
## App
|
||||
|
||||
- Chart
|
||||
- Added double click option on a legend to toggle the visibility of all other series
|
||||
@@ -270,14 +274,14 @@
|
||||
- Misc
|
||||
- Removed tracker even though it was a very privacy friendly as it appeared to not be working properly
|
||||
|
||||
### Price
|
||||
## Price
|
||||
|
||||
- Deleted old price datasets and their backups
|
||||
|
||||
## v. 0.1.0 | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
|
||||
# [satonomics-v0.1.0](https://github.com/bitcoinresearchkit/brk/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
|
||||
|
||||

|
||||

|
||||
|
||||
## v. 0.0.X | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
|
||||
# satonomics-v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
|
||||
|
||||

|
||||

|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# Guidelines
|
||||
|
||||
## Parser
|
||||
|
||||
- Avoid floats as much as possible
|
||||
- Use structs like `WAmount` and `Price` for calculations
|
||||
- **Only** use `WAmount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
|
||||
- No `Arc`, `Rc`, `Mutex` even from third party libraries, they're slower
|
||||
@@ -0,0 +1,88 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.0.76"
|
||||
package.homepage = "https://bitcoinresearchkit.org"
|
||||
package.repository = "https://github.com/bitcoinresearchkit/brk"
|
||||
|
||||
[profile.release]
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
panic = "abort"
|
||||
|
||||
[profile.profiling]
|
||||
inherits = "release"
|
||||
debug = true
|
||||
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
|
||||
[workspace.dependencies]
|
||||
arc-swap = "1.7.1"
|
||||
axum = "0.8.4"
|
||||
bincode = { version = "2.0.1", features = ["serde"] }
|
||||
bitcoin = { version = "0.32.6", features = ["serde"] }
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
brk_bundler = { version = "0.0.76", path = "crates/brk_bundler" }
|
||||
brk_cli = { version = "0.0.76", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0.0.76", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0.0.76", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0.0.76", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0.0.76", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.0.76", path = "crates/brk_indexer" }
|
||||
brk_interface = { version = "0.0.76", path = "crates/brk_interface" }
|
||||
brk_logger = { version = "0.0.76", path = "crates/brk_logger" }
|
||||
brk_mcp = { version = "0.0.76", path = "crates/brk_mcp" }
|
||||
brk_parser = { version = "0.0.76", path = "crates/brk_parser" }
|
||||
brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]}
|
||||
# brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]}
|
||||
brk_server = { version = "0.0.76", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.0.76", path = "crates/brk_store" }
|
||||
brk_vec = { version = "0.0.76", path = "crates/brk_vec" }
|
||||
byteview = "=0.6.1"
|
||||
clap = { version = "4.5.40", features = ["string"] }
|
||||
clap_derive = "4.5.40"
|
||||
color-eyre = "0.6.5"
|
||||
derive_deref = "1.1.1"
|
||||
fjall = "2.11.1"
|
||||
jiff = "0.2.15"
|
||||
log = { version = "0.4.27" }
|
||||
minreq = { version = "2.14.0", features = ["https", "serde_json"] }
|
||||
rayon = "1.10.0"
|
||||
schemars = "1.0.4"
|
||||
serde = { version = "1.0.219" }
|
||||
serde_bytes = "0.11.17"
|
||||
serde_derive = "1.0.219"
|
||||
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
|
||||
tabled = "0.20.0"
|
||||
tokio = { version = "1.46.1", features = ["rt-multi-thread"] }
|
||||
zerocopy = { version = "0.8.26" }
|
||||
zerocopy-derive = "0.8.26"
|
||||
|
||||
[workspace.metadata.release]
|
||||
shared-version = true
|
||||
tag-name = "v{{version}}"
|
||||
pre-release-commit-message = "release: v{{version}}"
|
||||
tag-message = "release: v{{version}}"
|
||||
|
||||
[workspace.metadata.dist]
|
||||
cargo-dist-version = "0.28.0"
|
||||
ci = "github"
|
||||
installers = []
|
||||
targets = [
|
||||
"aarch64-apple-darwin",
|
||||
"aarch64-unknown-linux-gnu",
|
||||
"x86_64-apple-darwin",
|
||||
"x86_64-unknown-linux-gnu",
|
||||
]
|
||||
|
||||
[workspace.metadata.dist.github-custom-runners]
|
||||
global = "ubuntu-latest"
|
||||
aarch64-apple-darwin.runner = "macos-14"
|
||||
x86_64-unknown-linux-gnu.runner = "ubuntu-latest"
|
||||
x86_64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
|
||||
aarch64-unknown-linux-gnu.runner = "ubuntu-latest"
|
||||
aarch64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 kibō
|
||||
Copyright (c) 2025 bitcoinresearchkit, kibo.money, satonomics
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,166 +1,107 @@
|
||||
<a href="https://kibo.money" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/kibo-money/kibo/main/assets/logo-long-text-dark.svg">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/kibo-money/kibo/main/assets/logo-long-text-light.svg">
|
||||
<img alt="kibō" src="https://raw.githubusercontent.com/kibo-money/kibo/main/assets/logo-long-text-light.svg" width="210" height="auto">
|
||||
</picture>
|
||||
</a>
|
||||
# Bitcoin Research Kit
|
||||
|
||||
## Description
|
||||
<p align="left">
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
<a href="https://crates.io/crates/brk">
|
||||
<img src="https://img.shields.io/crates/v/brk" alt="Version" />
|
||||
</a>
|
||||
<a href="https://docs.rs/brk">
|
||||
<img src="https://img.shields.io/docsrs/brk" alt="Documentation" />
|
||||
</a>
|
||||
<img src="https://img.shields.io/crates/size/brk" alt="Size" />
|
||||
<a href="https://deps.rs/crate/brk">
|
||||
<img src="https://deps.rs/crate/brk/latest/status.svg" alt="Dependency status">
|
||||
</a>
|
||||
<a href="https://discord.gg/HaR3wpH3nr">
|
||||
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
|
||||
</a>
|
||||
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
|
||||
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
|
||||
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
|
||||
</a>
|
||||
<a href="https://x.com/brkdotorg">
|
||||
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
|
||||
</a>
|
||||
</p>
|
||||
|
||||
[**kibō**](https://kibo.money) (_hope_ in japanese) is primarily an open source Bitcoin Core data extractor and visualizer (similar to [Glassnode](https://glassnode.com)) which goal is to empower anybody with data about Bitcoin for free.
|
||||
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network.
|
||||
|
||||
The project is split in 3 parts:
|
||||
In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) (soon) and [electrs](https://github.com/romanz/electrs) (soon) all in one package with a particular focus on simplicity and ease of use.
|
||||
|
||||
- First you have the extractor (parser), which parses the block data files from your Bitcoin Core node and computes a very wide range of datasets which are stored in compressed binary files
|
||||
> For the curious, it takes at the very least 24 hours to parse all the blocks and compute all datasets. After that it will wait for a new block and take between 1 and 3 minutes to be up to date
|
||||
- Then there is the website on which you can view, among other things, all datasets in various charts
|
||||
- Finally there is the server which serves the website and the generated data via an [API](https://github.com/kibo-money/kibo/tree/main#endpoints)
|
||||
The toolkit can be used in various ways to accommodate as many needs as possible:
|
||||
|
||||
Whether you're an enthusiast, a researcher, a miner, an analyst, a trader, a skeptic or just curious, there is something for everyone !
|
||||
- **[Website](https://bitcoinresearchkit.org)** \
|
||||
Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
|
||||
It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
|
||||
Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
|
||||
- **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
|
||||
Researchers and developers are free to use BRK's public API with  dataset variants at their disposal. \
|
||||
Just like the website, it's entirely free, with no authentication or rate-limiting.
|
||||
- **[AI](https://github.com/bitcoinresearchkit/brk/blob/main/crates/brk_mcp/README.md#brk-mcp)** \
|
||||
LLMs have to possibility to connect to BRK's backend through a [MCP](https://modelcontextprotocol.io/introduction). \
|
||||
It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts. \
|
||||
One-shot output examples: [Document](https://claude.ai/public/artifacts/71194d29-f965-417c-ba09-fdf0e4ecb1d5) // [Dashboard](https://claude.ai/public/artifacts/beef143f-399a-4ed4-b8bf-c986b776de42) // [Dashboard 2](https://claude.ai/public/artifacts/5430ae49-bb3d-4fc1-ab24-f1e33deb40dc)
|
||||
- **[CLI](https://crates.io/crates/brk_cli)** \
|
||||
Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \
|
||||
The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end.
|
||||
- **[Crates](https://crates.io/crates/brk)** \
|
||||
Rust developers have access to a wide range crates, each built upon one another with its own specific purpose, enabling independent use and offering great flexibility.
|
||||
PRs are welcome, especially if their goal is to introduce additional datasets.
|
||||
|
||||
This project was created out of frustration by all the alternatives that were either very expensive and thus discriminatory and against bitcoin values or just very limited and none were open-source and verifiable. So while it's not the first tool trying to solve these problems, it's the first that is completely free, open-source and self-hostable.
|
||||
The primary goal of this project is to be fully-featured and accessible for everyone, regardless of their background or financial situation - whether that person is an enthusiast, researcher, miner, analyst, or simply curious.
|
||||
|
||||
If you are a user of [mempool.space](https://mempool.space), you'll find this to be very complimentary, as it offers a macro view of the chain over time instead of a detailed one.
|
||||
In contrast, existing alternatives tend to be either [very costly](https://studio.glassnode.com/pricing) or missing essential features, with the vast majority being closed-source and unverifiable, which fundamentally undermines the principles of Bitcoin.
|
||||
|
||||
## Instances
|
||||
## Crates
|
||||
|
||||
| URL | Type | Version | Status | Last Height | Up Time Ratio |
|
||||
| ------------------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| [kibo.money](https://kibo.money) | Main |  |  |  |  |
|
||||
| [backup.kibo.money](https://backup.kibo.money) | Backup |  |  |  |  |
|
||||
| [preview.kibo.money](https://preview.kibo.money) | Dev |  |  |  |  |
|
||||
- [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates
|
||||
- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a Bitcoin Research Kit instance
|
||||
- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer
|
||||
- [`brk_core`](https://crates.io/crates/brk_core): The Core (Structs and Errors) of the Bitcoin Research Kit
|
||||
- [`brk_exit`](https://crates.io/crates/brk_exit): An exit blocker built on top of ctrlc
|
||||
- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher
|
||||
- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser
|
||||
- [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit
|
||||
- [`brk_mcp`](https://crates.io/crates/brk_mcp): A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK
|
||||
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
|
||||
- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine
|
||||
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
|
||||
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
|
||||
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
|
||||
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
|
||||
|
||||
Please open an issue if you want to add another instance
|
||||
## Hosting as a service
|
||||
|
||||
## Endpoints
|
||||
If you'd like to have your own instance hosted for you please contact [hosting@bitcoinresearchkit.org](mailto:hosting@bitcoinresearchkit.org).
|
||||
|
||||
> If you running locally, you can replace `https://kibo.money` by `http://localhost:3110`
|
||||
- 2 separate dedicated servers (1 GB/s each) with different ISPs and Cloudflare integration for enhanced performance and optimal availability
|
||||
- 99.99% SLA
|
||||
- Configured for speed
|
||||
- Updates delivered at your convenience
|
||||
- Direct communication for feature requests and support
|
||||
- Bitcoin Core or Knots with desired version
|
||||
- Optional subdomains: `*.bitcoinresearchkit.org`, `*.kibo.money` and `*.satonomics.xyz`
|
||||
- Logo featured in the Readme if desired
|
||||
|
||||
- [/](https://kibo.money/): Website
|
||||
- [/api](https://kibo.money/api): A JSON with all available datasets, with their respective id and endpoint, better viewed in a Firefox based browser
|
||||
- /api/TIMESCALE-to-ID: `TIMESCALE` can be `date` or `height`, and `ID` is the id with `_` replaced by `-`, let's take `date-to-close` (price at the end of each day) as an example
|
||||
- [/api/date-to-close](https://kibo.money/api/date-to-close): current year's values in a json format
|
||||
- [/api/date-to-close?chunk=2009](https://kibo.money/api/date-to-close?chunk=2009): values from the year 2009 in a json format
|
||||
- [/api/date-to-close?all=true](https://kibo.money/api/date-to-close?all=true): all values in a json format
|
||||
- You can also specify the extension to download a file, either `.json` or `.csv` to get the dataset in a CSV format; like so:
|
||||
- [/api/date-to-close.csv](https://kibo.money/api/date-to-close.csv)
|
||||
- [/api/date-to-close.csv?chunk=2009](https://kibo.money/api/date-to-close.csv?chunk=2009)
|
||||
- [/api/date-to-close.csv?all=true](https://kibo.money/api/date-to-close.csv?all=true)
|
||||
Pricing: `0.01 BTC / month` *or* `0.1 BTC / year`
|
||||
|
||||
## Roadmap
|
||||
## Acknowledgments
|
||||
|
||||
- **More Datasets/Charts**
|
||||
- **Simulations**
|
||||
- **Nostr integration**
|
||||
- **API Documentation**
|
||||
- **Descriptions**
|
||||
- **Docker support**
|
||||
- **Start9 support**
|
||||
Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
|
||||
|
||||
## Setup
|
||||
|
||||
### Requirements
|
||||
|
||||
- At least 16 GB of RAM
|
||||
- 1 TB of free space (will use 70% of that without defragmentation and 40% after)
|
||||
- A running instance of bitcoin-core with:
|
||||
- `-txindex=1`
|
||||
- `-blocksxor=0`
|
||||
- RPC credentials
|
||||
- Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly -txindex=1 -blocksxor=0`
|
||||
- Git
|
||||
|
||||
### Manual
|
||||
|
||||
_Mac OS and Linux only, Windows is unsupported_
|
||||
|
||||
First we need to install Rust (https://www.rust-lang.org/tools/install)
|
||||
|
||||
```bash
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
```
|
||||
|
||||
If you already had Rust installed you could update it just in case
|
||||
|
||||
```bash
|
||||
rustup update
|
||||
```
|
||||
|
||||
> If you're on Ubuntu you'll probably also need to install `open-ssl` with
|
||||
>
|
||||
> ```bash
|
||||
> sudo apt install libssl-dev pkg-config
|
||||
> ```
|
||||
|
||||
Optionally, you can also install `cargo-watch` for the server to automatically restart it on file change, which will be triggered by new code and new datasets from the parser (https://github.com/watchexec/cargo-watch?tab=readme-ov-file#install)
|
||||
|
||||
```bash
|
||||
cargo install cargo-watch --locked
|
||||
```
|
||||
|
||||
Then you need to choose a path where all files related to **kibō** will live
|
||||
|
||||
```bash
|
||||
cd ???
|
||||
```
|
||||
|
||||
We can now clone the repository
|
||||
|
||||
```bash
|
||||
git clone https://github.com/kibo-money/kibo.git
|
||||
```
|
||||
|
||||
In a new terminal, go to the `parser`'s folder of the repository
|
||||
|
||||
```bash
|
||||
cd ???/kibo/parser
|
||||
```
|
||||
|
||||
Now we can finally start by running the parser, you need to use the `./run.sh` script instead of `cargo run -r` as we need to set various system variables for the program to run smoothly
|
||||
|
||||
For the first launch, the parser will need several information such as:
|
||||
|
||||
- `--datadir`: which is bitcoin data directory path, prefer `$HOME` to `~` as the latter might not work
|
||||
|
||||
Optionally you can also specify:
|
||||
|
||||
- `--rpccookiefile`: the path to the cookie file if not default
|
||||
- `--rpcuser`: the username of the RPC credentials to talk to the bitcoin server if set
|
||||
- `--rpcpassword`: the password of the RPC credentials if set
|
||||
- `--rpcconnect`: if the bitcoin core server's IP is different than `localhost`
|
||||
- `--rpcport`: if the port is different than `8332`
|
||||
|
||||
Everything will be saved in a `config.toml` file, which will allow you to simply run `./run.sh` next time
|
||||
|
||||
Here's an example
|
||||
|
||||
```bash
|
||||
./run.sh --datadir=$HOME/Developer/bitcoin
|
||||
```
|
||||
|
||||
In a **new** terminal, go to the `server`'s folder of the repository
|
||||
|
||||
```bash
|
||||
cd ???/kibo/server
|
||||
```
|
||||
|
||||
And start it also with the `run.sh` script instead of `cargo run -r`
|
||||
|
||||
```bash
|
||||
./run.sh
|
||||
```
|
||||
|
||||
Then the easiest to let others access your server is to use `cloudflared` which will also cache requests. For more information go to: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/
|
||||
Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [bitcoinresearchkit.org](https://bitcoinresearchkit.org) public instance.
|
||||
|
||||
## Donate
|
||||
|
||||
<img width="159" alt="image" src="https://github.com/user-attachments/assets/8bbb759f-4874-46cb-b093-b30cb30f5828">
|
||||
[`bc1q09 8zsm89 m7kgyz e338vf ejhpdt 92ua9p 3peuve`](bitcoin:bc1q098zsm89m7kgyze338vfejhpdt92ua9p3peuve)
|
||||
|
||||
[bc1q950q4ukpxxm6wjjkv6cpq8jzpazaxrrwftctkt](bitcoin:bc1q950q4ukpxxm6wjjkv6cpq8jzpazaxrrwftctkt)
|
||||
[`lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4`](lightning:lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4)
|
||||
|
||||
<img width="159" alt="image" src="https://github.com/user-attachments/assets/745e39c7-be26-4f2a-90f2-54786e62ba35">
|
||||
|
||||
[lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4](lightning:lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4)
|
||||
|
||||
[Geyser Fund](https://geyser.fund/project/kibo/)
|
||||
[Geyser Fund](https://geyser.fund/project/brk)
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# TODO
|
||||
|
||||
- __crates__
|
||||
- _cli_
|
||||
- check disk space on first launch
|
||||
- add custom path support for config.toml
|
||||
- maybe add bitcoind download and launch support
|
||||
- via: https://github.com/rust-bitcoin/corepc/blob/master/node
|
||||
- test read/write speed, add warning if too low (<2gb/s)
|
||||
- pull latest version and notify is out of date
|
||||
- _computer_
|
||||
- **add rollback of states (in stateful)**
|
||||
- add support for per index computation
|
||||
- fix feerate which is always ZERO due to coinbase transaction
|
||||
- before computing multiple sources check their length, panic if not equal
|
||||
- add oracle price dataset (https://utxo.live/oracle/UTXOracle.py)
|
||||
- add address counts relative to all datasets
|
||||
- make decade, quarter, year datasets `computed` instead of `eager`
|
||||
- add 6 months (semester) interval datasets to builder
|
||||
- add revived/sent supply datasets
|
||||
- add `in-sats` version of all price datasets (average and co)
|
||||
- add `p2pk` group (sum of `p2pk33` and `p2pk65`)
|
||||
- add more date ranges (3-6 months and more)
|
||||
- add puell multiple dataset
|
||||
- add pi cycle dataset
|
||||
- add ema of price
|
||||
- add 7d and 30d ema to sell side risk ratio and sopr
|
||||
- add all possible charts from:
|
||||
- https://mainnet.observer
|
||||
- https://glassnode.com
|
||||
- https://checkonchain.com
|
||||
- https://researchbitcoin.net/exciting-update-coming-to-the-bitcoin-lab/
|
||||
- https://mempool.space/research
|
||||
- _indexer_
|
||||
- parse only the needed block number
|
||||
- maybe using https://developer.bitcoin.org/reference/rpc/getblockhash.html
|
||||
- _interface_
|
||||
- create pagination enum
|
||||
- from to
|
||||
- from option<count>
|
||||
- to option<count>
|
||||
- page + option<per page> default 1000 max 1000
|
||||
- from/to/count params don’t cap all combinations
|
||||
- example: from -10,000 count 10, won’t work if underlying vec isn’t 10k or more long
|
||||
- _parser_
|
||||
- save `vec` file instead of `json`
|
||||
- support lock file, process in read only if already opened in write mode
|
||||
- if less than X (10 maybe ?) get block using rpc instead of parsing the block files
|
||||
- _server_
|
||||
- api
|
||||
- add extensions support (.json .csv …)
|
||||
- if format instead of extension then don't download file
|
||||
- _vec_
|
||||
- add native lock file support (once it's available in stable rust)
|
||||
- improve compressed mode (slow reads)
|
||||
- add ema support
|
||||
- __docs__
|
||||
- _README_
|
||||
- add a comparison table with alternatives
|
||||
- add contribution section where help is needed
|
||||
- documentation/mcp/datasets/different front ends
|
||||
- add faq
|
||||
- __websites__
|
||||
- _default_
|
||||
- explorer
|
||||
- blocks
|
||||
- transactions
|
||||
- addresses
|
||||
- miners
|
||||
- maybe xpubs
|
||||
- charts
|
||||
- improve some names and colors
|
||||
- remove `sum` series when it's a duplicate of the `base` (in subsidy for example)
|
||||
- selected unit sometimes changes when going back end forth
|
||||
- add support for custom charts
|
||||
- price scale format depends on unit, hide digits for sats for example (if/when possible)
|
||||
- table
|
||||
- pagination
|
||||
- exports (.json, .csv,…)
|
||||
- search
|
||||
- datasets add legend, and keywords ?
|
||||
- height/address/txid
|
||||
- api
|
||||
- add api page with interactivity
|
||||
- global
|
||||
- **fix navigation/history**
|
||||
- move share button to footer ?
|
||||
- Use `ichart.createPane()` in wrapper
|
||||
- improve behavior when local storage is unavailable
|
||||
- by having a global state
|
||||
- __global__
|
||||
- check `TODO`s in codebase
|
||||
|
After Width: | Height: | Size: 340 KiB |
|
After Width: | Height: | Size: 94 KiB |
|
After Width: | Height: | Size: 78 KiB |
|
After Width: | Height: | Size: 133 KiB |
|
After Width: | Height: | Size: 263 KiB |
|
Before Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 208 KiB After Width: | Height: | Size: 208 KiB |
|
Before Width: | Height: | Size: 386 KiB After Width: | Height: | Size: 386 KiB |
@@ -1,4 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M 10.874 7.57 L 10.874 6.802 C 10.104 5.809 9.587 4.634 9.397 3.379 C 9.339 3.009 8.877 2.865 8.637 3.152 C 8.059 3.833 7.605 4.632 7.299 5.517 C 8.234 6.565 9.486 7.284 10.874 7.57 Z M 13.937 4.749 C 12.729 4.749 11.751 5.731 11.751 6.939 L 11.751 8.563 C 8.895 8.394 6.474 6.636 5.379 4.142 C 5.229 3.8 4.746 3.78 4.585 4.117 C 4.131 5.077 3.876 6.149 3.876 7.28 C 3.876 9.217 4.808 11.025 6.203 12.364 C 6.562 12.711 6.915 12.998 7.266 13.261 L 3.331 14.245 C 3.038 14.318 2.907 14.658 3.072 14.912 C 3.547 15.648 4.723 16.895 7.261 16.999 C 7.48 17.007 7.698 16.928 7.864 16.783 L 9.648 15.249 L 11.751 15.249 C 14.167 15.249 16.125 13.293 16.125 10.876 L 16.125 6.499 L 17 4.749 L 13.937 4.749 Z M 13.937 7.391 C 13.697 7.391 13.458 7.175 13.458 6.935 C 13.458 6.695 13.697 6.483 13.937 6.483 C 14.177 6.483 14.399 6.703 14.399 6.943 C 14.399 7.183 14.177 7.391 13.937 7.391 Z" style="fill: #12100f;"></path>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.0 KiB |
@@ -1,4 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M 10.874 7.57 L 10.874 6.802 C 10.104 5.809 9.587 4.634 9.397 3.379 C 9.339 3.009 8.877 2.865 8.637 3.152 C 8.059 3.833 7.605 4.632 7.299 5.517 C 8.234 6.565 9.486 7.284 10.874 7.57 Z M 13.937 4.749 C 12.729 4.749 11.751 5.731 11.751 6.939 L 11.751 8.563 C 8.895 8.394 6.474 6.636 5.379 4.142 C 5.229 3.8 4.746 3.78 4.585 4.117 C 4.131 5.077 3.876 6.149 3.876 7.28 C 3.876 9.217 4.808 11.025 6.203 12.364 C 6.562 12.711 6.915 12.998 7.266 13.261 L 3.331 14.245 C 3.038 14.318 2.907 14.658 3.072 14.912 C 3.547 15.648 4.723 16.895 7.261 16.999 C 7.48 17.007 7.698 16.928 7.864 16.783 L 9.648 15.249 L 11.751 15.249 C 14.167 15.249 16.125 13.293 16.125 10.876 L 16.125 6.499 L 17 4.749 L 13.937 4.749 Z M 13.937 7.391 C 13.697 7.391 13.458 7.175 13.458 6.935 C 13.458 6.695 13.697 6.483 13.937 6.483 C 14.177 6.483 14.399 6.703 14.399 6.943 C 14.399 7.183 14.177 7.391 13.937 7.391 Z" style="fill: #fffaf6;"></path>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.0 KiB |
@@ -1,4 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M 10.874 7.57 L 10.874 6.802 C 10.104 5.809 9.587 4.634 9.397 3.379 C 9.339 3.009 8.877 2.865 8.637 3.152 C 8.059 3.833 7.605 4.632 7.299 5.517 C 8.234 6.565 9.486 7.284 10.874 7.57 Z M 13.937 4.749 C 12.729 4.749 11.751 5.731 11.751 6.939 L 11.751 8.563 C 8.895 8.394 6.474 6.636 5.379 4.142 C 5.229 3.8 4.746 3.78 4.585 4.117 C 4.131 5.077 3.876 6.149 3.876 7.28 C 3.876 9.217 4.808 11.025 6.203 12.364 C 6.562 12.711 6.915 12.998 7.266 13.261 L 3.331 14.245 C 3.038 14.318 2.907 14.658 3.072 14.912 C 3.547 15.648 4.723 16.895 7.261 16.999 C 7.48 17.007 7.698 16.928 7.864 16.783 L 9.648 15.249 L 11.751 15.249 C 14.167 15.249 16.125 13.293 16.125 10.876 L 16.125 6.499 L 17 4.749 L 13.937 4.749 Z M 13.937 7.391 C 13.697 7.391 13.458 7.175 13.458 6.935 C 13.458 6.695 13.697 6.483 13.937 6.483 C 14.177 6.483 14.399 6.703 14.399 6.943 C 14.399 7.183 14.177 7.391 13.937 7.391 Z" style="fill: #f26610;"></path>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.0 KiB |
@@ -1,11 +0,0 @@
|
||||
<svg viewBox="0 0 720 180" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs></defs>
|
||||
<g transform="matrix(7.5, 0, 0, 7.5, -2046.71228, -1592.744873)">
|
||||
<ellipse style="fill: #f26610;" cx="284.895" cy="224.366" rx="12" ry="12"></ellipse>
|
||||
<path d="M 285.769 221.936 L 285.769 221.168 C 284.999 220.175 284.482 219 284.292 217.745 C 284.234 217.375 283.772 217.231 283.532 217.518 C 282.954 218.199 282.5 218.998 282.194 219.883 C 283.129 220.931 284.381 221.65 285.769 221.936 Z M 288.832 219.115 C 287.624 219.115 286.646 220.097 286.646 221.305 L 286.646 222.929 C 283.79 222.76 281.369 221.002 280.274 218.508 C 280.124 218.166 279.641 218.146 279.48 218.483 C 279.026 219.443 278.771 220.515 278.771 221.646 C 278.771 223.583 279.703 225.391 281.098 226.73 C 281.457 227.077 281.81 227.364 282.161 227.627 L 278.226 228.611 C 277.933 228.684 277.802 229.024 277.967 229.278 C 278.442 230.014 279.618 231.261 282.156 231.365 C 282.375 231.373 282.593 231.294 282.759 231.149 L 284.543 229.615 L 286.646 229.615 C 289.062 229.615 291.02 227.659 291.02 225.242 L 291.02 220.865 L 291.895 219.115 L 288.832 219.115 Z M 288.832 221.757 C 288.592 221.757 288.353 221.541 288.353 221.301 C 288.353 221.061 288.592 220.849 288.832 220.849 C 289.072 220.849 289.294 221.069 289.294 221.309 C 289.294 221.549 289.072 221.757 288.832 221.757 Z" style="fill: #fffaf6;"></path>
|
||||
</g>
|
||||
<g transform="matrix(1, 0, 0, 1, -30, 0)">
|
||||
<path d="M 278.049 146.789 L 278.049 127.527 L 287.141 117.972 L 304.4 146.789 L 331.83 146.789 L 303.784 100.251 L 332.755 69.739 L 303.013 69.739 L 278.049 97.477 L 278.049 30.598 L 254.318 30.598 L 254.318 146.789 L 278.049 146.789 Z M 354.169 57.719 C 361.565 57.719 367.575 51.709 367.575 44.158 C 367.575 36.608 361.565 30.752 354.169 30.752 C 346.618 30.752 340.608 36.608 340.608 44.158 C 340.608 51.709 346.618 57.719 354.169 57.719 Z M 342.457 146.789 L 366.188 146.789 L 366.188 69.739 L 342.457 69.739 L 342.457 146.789 Z M 406.407 146.789 L 407.64 136.927 C 411.801 144.015 421.047 148.792 431.834 148.792 C 453.716 148.792 468.972 132.92 468.972 109.035 C 468.972 83.916 455.257 67.119 433.683 67.119 C 422.588 67.119 412.417 71.742 407.794 78.677 L 407.794 30.598 L 384.063 30.598 L 384.063 146.789 L 406.407 146.789 Z M 407.948 107.802 C 407.948 96.244 415.653 88.539 426.749 88.539 C 437.998 88.539 445.087 96.398 445.087 107.802 C 445.087 119.205 437.998 127.064 426.749 127.064 C 415.653 127.064 407.948 119.359 407.948 107.802 Z M 498.713 56.332 L 543.402 56.332 L 543.402 40.306 L 498.713 40.306 L 498.713 56.332 Z M 478.526 108.11 C 478.526 132.458 496.402 148.638 521.058 148.638 C 545.56 148.638 563.435 132.458 563.435 108.11 C 563.435 83.762 545.56 67.428 521.058 67.428 C 496.402 67.428 478.526 83.762 478.526 108.11 Z M 502.412 107.956 C 502.412 96.398 509.963 88.693 521.058 88.693 C 531.999 88.693 539.55 96.398 539.55 107.956 C 539.55 119.667 531.999 127.372 521.058 127.372 C 509.963 127.372 502.412 119.667 502.412 107.956 Z" style="fill: #fffaf6;"></path>
|
||||
<path d="M 589.19 97.802 L 589.19 106.23 L 610.948 106.23 C 605.1 112.938 597.446 119.044 587.986 124.376 L 593.404 131.514 C 597.532 128.934 601.488 126.268 605.186 123.43 L 605.186 146.048 L 614.13 146.048 L 614.13 123.43 L 626.944 123.43 L 626.944 149.402 L 635.974 149.402 L 635.974 123.43 L 649.82 123.43 L 649.82 134.008 C 649.82 136.072 649.046 137.104 647.498 137.104 L 640.36 136.674 L 642.768 145.188 L 650.422 145.188 C 655.926 145.188 658.678 142.092 658.678 135.986 L 658.678 115.174 L 635.974 115.174 L 635.974 108.638 L 626.944 108.638 L 626.944 115.174 L 614.388 115.174 C 617.054 112.336 619.548 109.326 621.784 106.23 L 665.128 106.23 L 665.128 97.802 L 626.858 97.802 C 627.89 95.824 628.836 93.76 629.696 91.61 L 620.838 90.492 C 619.806 92.9 618.516 95.394 617.14 97.802 L 589.19 97.802 Z M 648.1 68.734 C 642.338 72.088 636.232 75.098 629.868 77.678 C 621.612 75.012 612.926 72.518 603.896 70.282 L 599.252 77.248 C 605.272 78.624 611.206 80.258 617.226 82.15 C 610.088 84.386 602.606 86.106 594.78 87.482 L 599.596 95.308 C 612.324 92.04 622.472 89.116 630.04 86.364 C 638.124 89.116 646.122 92.298 654.034 95.824 L 658.936 88.428 C 653.26 86.02 647.412 83.698 641.392 81.548 C 646.208 79.226 651.11 76.56 655.926 73.55 L 648.1 68.734 Z M 675.438 77.85 L 675.438 85.848 L 682.404 85.848 L 682.404 98.92 C 682.404 101.5 681.114 103.22 678.62 104.166 L 680.684 110.874 C 692.036 108.896 701.926 106.66 710.182 104.08 L 708.634 96.426 C 703.474 98.146 697.454 99.608 690.574 100.984 L 690.574 85.848 L 712.332 85.848 L 712.332 77.85 L 698.916 77.85 C 698.4 74.668 697.884 71.744 697.368 69.164 L 688.338 70.712 C 688.94 72.862 689.542 75.27 690.144 77.85 L 675.438 77.85 Z M 724.028 89.632 L 739.25 89.632 L 739.25 93.502 L 723.856 93.502 C 723.942 92.47 724.028 91.352 724.028 90.32 L 724.028 89.632 Z M 739.25 83.096 L 724.028 83.096 L 724.028 79.226 L 739.25 79.226 L 739.25 83.096 Z M 722.652 100.038 L 739.25 100.038 L 739.25 100.898 C 739.25 103.048 738.218 104.166 736.24 104.166 C 733.918 104.166 731.424 103.994 728.758 103.822 L 730.822 111.562 L 738.734 111.562 C 744.582 111.562 747.506 108.982 747.506 103.908 L 747.506 72.002 L 715.6 72.002 L 715.6 90.922 C 715.428 97.286 713.192 102.532 708.892 106.746 L 715.342 112.594 C 718.782 109.068 721.276 104.854 722.652 100.038 Z M 708.462 121.452 L 708.462 126.784 L 683.608 126.784 L 683.608 134.352 L 708.462 134.352 L 708.462 139.598 L 675.524 139.598 L 675.524 147.51 L 750 147.51 L 750 139.598 L 717.062 139.598 L 717.062 134.352 L 742.174 134.352 L 742.174 126.784 L 717.062 126.784 L 717.062 121.452 L 746.216 121.452 L 746.216 113.712 L 679.308 113.712 L 679.308 121.452 L 708.462 121.452 Z" style="fill: #68625f"></path>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 5.6 KiB |
@@ -1,11 +0,0 @@
|
||||
<svg viewBox="0 0 720 180" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs></defs>
|
||||
<g transform="matrix(7.5, 0, 0, 7.5, -2046.71228, -1592.744873)">
|
||||
<ellipse style="fill: #f26610;" cx="284.895" cy="224.366" rx="12" ry="12"></ellipse>
|
||||
<path d="M 285.769 221.936 L 285.769 221.168 C 284.999 220.175 284.482 219 284.292 217.745 C 284.234 217.375 283.772 217.231 283.532 217.518 C 282.954 218.199 282.5 218.998 282.194 219.883 C 283.129 220.931 284.381 221.65 285.769 221.936 Z M 288.832 219.115 C 287.624 219.115 286.646 220.097 286.646 221.305 L 286.646 222.929 C 283.79 222.76 281.369 221.002 280.274 218.508 C 280.124 218.166 279.641 218.146 279.48 218.483 C 279.026 219.443 278.771 220.515 278.771 221.646 C 278.771 223.583 279.703 225.391 281.098 226.73 C 281.457 227.077 281.81 227.364 282.161 227.627 L 278.226 228.611 C 277.933 228.684 277.802 229.024 277.967 229.278 C 278.442 230.014 279.618 231.261 282.156 231.365 C 282.375 231.373 282.593 231.294 282.759 231.149 L 284.543 229.615 L 286.646 229.615 C 289.062 229.615 291.02 227.659 291.02 225.242 L 291.02 220.865 L 291.895 219.115 L 288.832 219.115 Z M 288.832 221.757 C 288.592 221.757 288.353 221.541 288.353 221.301 C 288.353 221.061 288.592 220.849 288.832 220.849 C 289.072 220.849 289.294 221.069 289.294 221.309 C 289.294 221.549 289.072 221.757 288.832 221.757 Z" style="fill: #fffaf6;"></path>
|
||||
</g>
|
||||
<g transform="matrix(1, 0, 0, 1, -30, 0)">
|
||||
<path d="M 278.049 146.789 L 278.049 127.527 L 287.141 117.972 L 304.4 146.789 L 331.83 146.789 L 303.784 100.251 L 332.755 69.739 L 303.013 69.739 L 278.049 97.477 L 278.049 30.598 L 254.318 30.598 L 254.318 146.789 L 278.049 146.789 Z M 354.169 57.719 C 361.565 57.719 367.575 51.709 367.575 44.158 C 367.575 36.608 361.565 30.752 354.169 30.752 C 346.618 30.752 340.608 36.608 340.608 44.158 C 340.608 51.709 346.618 57.719 354.169 57.719 Z M 342.457 146.789 L 366.188 146.789 L 366.188 69.739 L 342.457 69.739 L 342.457 146.789 Z M 406.407 146.789 L 407.64 136.927 C 411.801 144.015 421.047 148.792 431.834 148.792 C 453.716 148.792 468.972 132.92 468.972 109.035 C 468.972 83.916 455.257 67.119 433.683 67.119 C 422.588 67.119 412.417 71.742 407.794 78.677 L 407.794 30.598 L 384.063 30.598 L 384.063 146.789 L 406.407 146.789 Z M 407.948 107.802 C 407.948 96.244 415.653 88.539 426.749 88.539 C 437.998 88.539 445.087 96.398 445.087 107.802 C 445.087 119.205 437.998 127.064 426.749 127.064 C 415.653 127.064 407.948 119.359 407.948 107.802 Z M 498.713 56.332 L 543.402 56.332 L 543.402 40.306 L 498.713 40.306 L 498.713 56.332 Z M 478.526 108.11 C 478.526 132.458 496.402 148.638 521.058 148.638 C 545.56 148.638 563.435 132.458 563.435 108.11 C 563.435 83.762 545.56 67.428 521.058 67.428 C 496.402 67.428 478.526 83.762 478.526 108.11 Z M 502.412 107.956 C 502.412 96.398 509.963 88.693 521.058 88.693 C 531.999 88.693 539.55 96.398 539.55 107.956 C 539.55 119.667 531.999 127.372 521.058 127.372 C 509.963 127.372 502.412 119.667 502.412 107.956 Z" style="fill: #12100f;"></path>
|
||||
<path d="M 589.19 97.802 L 589.19 106.23 L 610.948 106.23 C 605.1 112.938 597.446 119.044 587.986 124.376 L 593.404 131.514 C 597.532 128.934 601.488 126.268 605.186 123.43 L 605.186 146.048 L 614.13 146.048 L 614.13 123.43 L 626.944 123.43 L 626.944 149.402 L 635.974 149.402 L 635.974 123.43 L 649.82 123.43 L 649.82 134.008 C 649.82 136.072 649.046 137.104 647.498 137.104 L 640.36 136.674 L 642.768 145.188 L 650.422 145.188 C 655.926 145.188 658.678 142.092 658.678 135.986 L 658.678 115.174 L 635.974 115.174 L 635.974 108.638 L 626.944 108.638 L 626.944 115.174 L 614.388 115.174 C 617.054 112.336 619.548 109.326 621.784 106.23 L 665.128 106.23 L 665.128 97.802 L 626.858 97.802 C 627.89 95.824 628.836 93.76 629.696 91.61 L 620.838 90.492 C 619.806 92.9 618.516 95.394 617.14 97.802 L 589.19 97.802 Z M 648.1 68.734 C 642.338 72.088 636.232 75.098 629.868 77.678 C 621.612 75.012 612.926 72.518 603.896 70.282 L 599.252 77.248 C 605.272 78.624 611.206 80.258 617.226 82.15 C 610.088 84.386 602.606 86.106 594.78 87.482 L 599.596 95.308 C 612.324 92.04 622.472 89.116 630.04 86.364 C 638.124 89.116 646.122 92.298 654.034 95.824 L 658.936 88.428 C 653.26 86.02 647.412 83.698 641.392 81.548 C 646.208 79.226 651.11 76.56 655.926 73.55 L 648.1 68.734 Z M 675.438 77.85 L 675.438 85.848 L 682.404 85.848 L 682.404 98.92 C 682.404 101.5 681.114 103.22 678.62 104.166 L 680.684 110.874 C 692.036 108.896 701.926 106.66 710.182 104.08 L 708.634 96.426 C 703.474 98.146 697.454 99.608 690.574 100.984 L 690.574 85.848 L 712.332 85.848 L 712.332 77.85 L 698.916 77.85 C 698.4 74.668 697.884 71.744 697.368 69.164 L 688.338 70.712 C 688.94 72.862 689.542 75.27 690.144 77.85 L 675.438 77.85 Z M 724.028 89.632 L 739.25 89.632 L 739.25 93.502 L 723.856 93.502 C 723.942 92.47 724.028 91.352 724.028 90.32 L 724.028 89.632 Z M 739.25 83.096 L 724.028 83.096 L 724.028 79.226 L 739.25 79.226 L 739.25 83.096 Z M 722.652 100.038 L 739.25 100.038 L 739.25 100.898 C 739.25 103.048 738.218 104.166 736.24 104.166 C 733.918 104.166 731.424 103.994 728.758 103.822 L 730.822 111.562 L 738.734 111.562 C 744.582 111.562 747.506 108.982 747.506 103.908 L 747.506 72.002 L 715.6 72.002 L 715.6 90.922 C 715.428 97.286 713.192 102.532 708.892 106.746 L 715.342 112.594 C 718.782 109.068 721.276 104.854 722.652 100.038 Z M 708.462 121.452 L 708.462 126.784 L 683.608 126.784 L 683.608 134.352 L 708.462 134.352 L 708.462 139.598 L 675.524 139.598 L 675.524 147.51 L 750 147.51 L 750 139.598 L 717.062 139.598 L 717.062 134.352 L 742.174 134.352 L 742.174 126.784 L 717.062 126.784 L 717.062 121.452 L 746.216 121.452 L 746.216 113.712 L 679.308 113.712 L 679.308 121.452 L 708.462 121.452 Z" style="fill: #b4aca9;"></path>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 5.6 KiB |
@@ -1,5 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
|
||||
<ellipse style="fill: #f26610;" cx="12" cy="12" rx="12" ry="12"/>
|
||||
<path d="M 12.874 9.57 L 12.874 8.802 C 12.104 7.809 11.587 6.634 11.397 5.379 C 11.339 5.009 10.877 4.865 10.637 5.152 C 10.059 5.833 9.605 6.632 9.299 7.517 C 10.234 8.565 11.486 9.284 12.874 9.57 Z M 15.937 6.749 C 14.729 6.749 13.751 7.731 13.751 8.939 L 13.751 10.563 C 10.895 10.394 8.474 8.636 7.379 6.142 C 7.229 5.8 6.746 5.78 6.585 6.117 C 6.131 7.077 5.876 8.149 5.876 9.28 C 5.876 11.217 6.808 13.025 8.203 14.364 C 8.562 14.711 8.915 14.998 9.266 15.261 L 5.331 16.245 C 5.038 16.318 4.907 16.658 5.072 16.912 C 5.547 17.648 6.723 18.895 9.261 18.999 C 9.48 19.007 9.698 18.928 9.864 18.783 L 11.648 17.249 L 13.751 17.249 C 16.167 17.249 18.125 15.293 18.125 12.876 L 18.125 8.499 L 19 6.749 L 15.937 6.749 Z M 15.937 9.391 C 15.697 9.391 15.458 9.175 15.458 8.935 C 15.458 8.695 15.697 8.483 15.937 8.483 C 16.177 8.483 16.399 8.703 16.399 8.943 C 16.399 9.183 16.177 9.391 15.937 9.391 Z" style="fill: #fffaf6;"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.1 KiB |
@@ -1,8 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 500 180" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs/>
|
||||
<g transform="matrix(1, 0, 0, 1, -252.158997, 0)">
|
||||
<path d="M 278.049 146.789 L 278.049 127.527 L 287.141 117.972 L 304.4 146.789 L 331.83 146.789 L 303.784 100.251 L 332.755 69.739 L 303.013 69.739 L 278.049 97.477 L 278.049 30.598 L 254.318 30.598 L 254.318 146.789 L 278.049 146.789 Z M 354.169 57.719 C 361.565 57.719 367.575 51.709 367.575 44.158 C 367.575 36.608 361.565 30.752 354.169 30.752 C 346.618 30.752 340.608 36.608 340.608 44.158 C 340.608 51.709 346.618 57.719 354.169 57.719 Z M 342.457 146.789 L 366.188 146.789 L 366.188 69.739 L 342.457 69.739 L 342.457 146.789 Z M 406.407 146.789 L 407.64 136.927 C 411.801 144.015 421.047 148.792 431.834 148.792 C 453.716 148.792 468.972 132.92 468.972 109.035 C 468.972 83.916 455.257 67.119 433.683 67.119 C 422.588 67.119 412.417 71.742 407.794 78.677 L 407.794 30.598 L 384.063 30.598 L 384.063 146.789 L 406.407 146.789 Z M 407.948 107.802 C 407.948 96.244 415.653 88.539 426.749 88.539 C 437.998 88.539 445.087 96.398 445.087 107.802 C 445.087 119.205 437.998 127.064 426.749 127.064 C 415.653 127.064 407.948 119.359 407.948 107.802 Z M 498.713 56.332 L 543.402 56.332 L 543.402 40.306 L 498.713 40.306 L 498.713 56.332 Z M 478.526 108.11 C 478.526 132.458 496.402 148.638 521.058 148.638 C 545.56 148.638 563.435 132.458 563.435 108.11 C 563.435 83.762 545.56 67.428 521.058 67.428 C 496.402 67.428 478.526 83.762 478.526 108.11 Z M 502.412 107.956 C 502.412 96.398 509.963 88.693 521.058 88.693 C 531.999 88.693 539.55 96.398 539.55 107.956 C 539.55 119.667 531.999 127.372 521.058 127.372 C 509.963 127.372 502.412 119.667 502.412 107.956 Z" style="fill: #fffaf6;"/>
|
||||
<path d="M 589.19 97.802 L 589.19 106.23 L 610.948 106.23 C 605.1 112.938 597.446 119.044 587.986 124.376 L 593.404 131.514 C 597.532 128.934 601.488 126.268 605.186 123.43 L 605.186 146.048 L 614.13 146.048 L 614.13 123.43 L 626.944 123.43 L 626.944 149.402 L 635.974 149.402 L 635.974 123.43 L 649.82 123.43 L 649.82 134.008 C 649.82 136.072 649.046 137.104 647.498 137.104 L 640.36 136.674 L 642.768 145.188 L 650.422 145.188 C 655.926 145.188 658.678 142.092 658.678 135.986 L 658.678 115.174 L 635.974 115.174 L 635.974 108.638 L 626.944 108.638 L 626.944 115.174 L 614.388 115.174 C 617.054 112.336 619.548 109.326 621.784 106.23 L 665.128 106.23 L 665.128 97.802 L 626.858 97.802 C 627.89 95.824 628.836 93.76 629.696 91.61 L 620.838 90.492 C 619.806 92.9 618.516 95.394 617.14 97.802 L 589.19 97.802 Z M 648.1 68.734 C 642.338 72.088 636.232 75.098 629.868 77.678 C 621.612 75.012 612.926 72.518 603.896 70.282 L 599.252 77.248 C 605.272 78.624 611.206 80.258 617.226 82.15 C 610.088 84.386 602.606 86.106 594.78 87.482 L 599.596 95.308 C 612.324 92.04 622.472 89.116 630.04 86.364 C 638.124 89.116 646.122 92.298 654.034 95.824 L 658.936 88.428 C 653.26 86.02 647.412 83.698 641.392 81.548 C 646.208 79.226 651.11 76.56 655.926 73.55 L 648.1 68.734 Z M 675.438 77.85 L 675.438 85.848 L 682.404 85.848 L 682.404 98.92 C 682.404 101.5 681.114 103.22 678.62 104.166 L 680.684 110.874 C 692.036 108.896 701.926 106.66 710.182 104.08 L 708.634 96.426 C 703.474 98.146 697.454 99.608 690.574 100.984 L 690.574 85.848 L 712.332 85.848 L 712.332 77.85 L 698.916 77.85 C 698.4 74.668 697.884 71.744 697.368 69.164 L 688.338 70.712 C 688.94 72.862 689.542 75.27 690.144 77.85 L 675.438 77.85 Z M 724.028 89.632 L 739.25 89.632 L 739.25 93.502 L 723.856 93.502 C 723.942 92.47 724.028 91.352 724.028 90.32 L 724.028 89.632 Z M 739.25 83.096 L 724.028 83.096 L 724.028 79.226 L 739.25 79.226 L 739.25 83.096 Z M 722.652 100.038 L 739.25 100.038 L 739.25 100.898 C 739.25 103.048 738.218 104.166 736.24 104.166 C 733.918 104.166 731.424 103.994 728.758 103.822 L 730.822 111.562 L 738.734 111.562 C 744.582 111.562 747.506 108.982 747.506 103.908 L 747.506 72.002 L 715.6 72.002 L 715.6 90.922 C 715.428 97.286 713.192 102.532 708.892 106.746 L 715.342 112.594 C 718.782 109.068 721.276 104.854 722.652 100.038 Z M 708.462 121.452 L 708.462 126.784 L 683.608 126.784 L 683.608 134.352 L 708.462 134.352 L 708.462 139.598 L 675.524 139.598 L 675.524 147.51 L 750 147.51 L 750 139.598 L 717.062 139.598 L 717.062 134.352 L 742.174 134.352 L 742.174 126.784 L 717.062 126.784 L 717.062 121.452 L 746.216 121.452 L 746.216 113.712 L 679.308 113.712 L 679.308 121.452 L 708.462 121.452 Z" style="fill: #867e7b;"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 4.4 KiB |
@@ -1,8 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 500 180" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs/>
|
||||
<g transform="matrix(1, 0, 0, 1, -252.158997, 0)">
|
||||
<path d="M 278.049 146.789 L 278.049 127.527 L 287.141 117.972 L 304.4 146.789 L 331.83 146.789 L 303.784 100.251 L 332.755 69.739 L 303.013 69.739 L 278.049 97.477 L 278.049 30.598 L 254.318 30.598 L 254.318 146.789 L 278.049 146.789 Z M 354.169 57.719 C 361.565 57.719 367.575 51.709 367.575 44.158 C 367.575 36.608 361.565 30.752 354.169 30.752 C 346.618 30.752 340.608 36.608 340.608 44.158 C 340.608 51.709 346.618 57.719 354.169 57.719 Z M 342.457 146.789 L 366.188 146.789 L 366.188 69.739 L 342.457 69.739 L 342.457 146.789 Z M 406.407 146.789 L 407.64 136.927 C 411.801 144.015 421.047 148.792 431.834 148.792 C 453.716 148.792 468.972 132.92 468.972 109.035 C 468.972 83.916 455.257 67.119 433.683 67.119 C 422.588 67.119 412.417 71.742 407.794 78.677 L 407.794 30.598 L 384.063 30.598 L 384.063 146.789 L 406.407 146.789 Z M 407.948 107.802 C 407.948 96.244 415.653 88.539 426.749 88.539 C 437.998 88.539 445.087 96.398 445.087 107.802 C 445.087 119.205 437.998 127.064 426.749 127.064 C 415.653 127.064 407.948 119.359 407.948 107.802 Z M 498.713 56.332 L 543.402 56.332 L 543.402 40.306 L 498.713 40.306 L 498.713 56.332 Z M 478.526 108.11 C 478.526 132.458 496.402 148.638 521.058 148.638 C 545.56 148.638 563.435 132.458 563.435 108.11 C 563.435 83.762 545.56 67.428 521.058 67.428 C 496.402 67.428 478.526 83.762 478.526 108.11 Z M 502.412 107.956 C 502.412 96.398 509.963 88.693 521.058 88.693 C 531.999 88.693 539.55 96.398 539.55 107.956 C 539.55 119.667 531.999 127.372 521.058 127.372 C 509.963 127.372 502.412 119.667 502.412 107.956 Z" style="fill: rgb(16, 16, 14);"/>
|
||||
<path d="M 589.19 97.802 L 589.19 106.23 L 610.948 106.23 C 605.1 112.938 597.446 119.044 587.986 124.376 L 593.404 131.514 C 597.532 128.934 601.488 126.268 605.186 123.43 L 605.186 146.048 L 614.13 146.048 L 614.13 123.43 L 626.944 123.43 L 626.944 149.402 L 635.974 149.402 L 635.974 123.43 L 649.82 123.43 L 649.82 134.008 C 649.82 136.072 649.046 137.104 647.498 137.104 L 640.36 136.674 L 642.768 145.188 L 650.422 145.188 C 655.926 145.188 658.678 142.092 658.678 135.986 L 658.678 115.174 L 635.974 115.174 L 635.974 108.638 L 626.944 108.638 L 626.944 115.174 L 614.388 115.174 C 617.054 112.336 619.548 109.326 621.784 106.23 L 665.128 106.23 L 665.128 97.802 L 626.858 97.802 C 627.89 95.824 628.836 93.76 629.696 91.61 L 620.838 90.492 C 619.806 92.9 618.516 95.394 617.14 97.802 L 589.19 97.802 Z M 648.1 68.734 C 642.338 72.088 636.232 75.098 629.868 77.678 C 621.612 75.012 612.926 72.518 603.896 70.282 L 599.252 77.248 C 605.272 78.624 611.206 80.258 617.226 82.15 C 610.088 84.386 602.606 86.106 594.78 87.482 L 599.596 95.308 C 612.324 92.04 622.472 89.116 630.04 86.364 C 638.124 89.116 646.122 92.298 654.034 95.824 L 658.936 88.428 C 653.26 86.02 647.412 83.698 641.392 81.548 C 646.208 79.226 651.11 76.56 655.926 73.55 L 648.1 68.734 Z M 675.438 77.85 L 675.438 85.848 L 682.404 85.848 L 682.404 98.92 C 682.404 101.5 681.114 103.22 678.62 104.166 L 680.684 110.874 C 692.036 108.896 701.926 106.66 710.182 104.08 L 708.634 96.426 C 703.474 98.146 697.454 99.608 690.574 100.984 L 690.574 85.848 L 712.332 85.848 L 712.332 77.85 L 698.916 77.85 C 698.4 74.668 697.884 71.744 697.368 69.164 L 688.338 70.712 C 688.94 72.862 689.542 75.27 690.144 77.85 L 675.438 77.85 Z M 724.028 89.632 L 739.25 89.632 L 739.25 93.502 L 723.856 93.502 C 723.942 92.47 724.028 91.352 724.028 90.32 L 724.028 89.632 Z M 739.25 83.096 L 724.028 83.096 L 724.028 79.226 L 739.25 79.226 L 739.25 83.096 Z M 722.652 100.038 L 739.25 100.038 L 739.25 100.898 C 739.25 103.048 738.218 104.166 736.24 104.166 C 733.918 104.166 731.424 103.994 728.758 103.822 L 730.822 111.562 L 738.734 111.562 C 744.582 111.562 747.506 108.982 747.506 103.908 L 747.506 72.002 L 715.6 72.002 L 715.6 90.922 C 715.428 97.286 713.192 102.532 708.892 106.746 L 715.342 112.594 C 718.782 109.068 721.276 104.854 722.652 100.038 Z M 708.462 121.452 L 708.462 126.784 L 683.608 126.784 L 683.608 134.352 L 708.462 134.352 L 708.462 139.598 L 675.524 139.598 L 675.524 147.51 L 750 147.51 L 750 139.598 L 717.062 139.598 L 717.062 134.352 L 742.174 134.352 L 742.174 126.784 L 717.062 126.784 L 717.062 121.452 L 746.216 121.452 L 746.216 113.712 L 679.308 113.712 L 679.308 121.452 L 708.462 121.452 Z" style="fill: rgb(192, 192, 171);"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 4.4 KiB |
@@ -1,7 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 310 180" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs/>
|
||||
<g transform="matrix(1, 0, 0, 1, -253.876495, 0)">
|
||||
<path d="M 278.049 146.789 L 278.049 127.527 L 287.141 117.972 L 304.4 146.789 L 331.83 146.789 L 303.784 100.251 L 332.755 69.739 L 303.013 69.739 L 278.049 97.477 L 278.049 30.598 L 254.318 30.598 L 254.318 146.789 L 278.049 146.789 Z M 354.169 57.719 C 361.565 57.719 367.575 51.709 367.575 44.158 C 367.575 36.608 361.565 30.752 354.169 30.752 C 346.618 30.752 340.608 36.608 340.608 44.158 C 340.608 51.709 346.618 57.719 354.169 57.719 Z M 342.457 146.789 L 366.188 146.789 L 366.188 69.739 L 342.457 69.739 L 342.457 146.789 Z M 406.407 146.789 L 407.64 136.927 C 411.801 144.015 421.047 148.792 431.834 148.792 C 453.716 148.792 468.972 132.92 468.972 109.035 C 468.972 83.916 455.257 67.119 433.683 67.119 C 422.588 67.119 412.417 71.742 407.794 78.677 L 407.794 30.598 L 384.063 30.598 L 384.063 146.789 L 406.407 146.789 Z M 407.948 107.802 C 407.948 96.244 415.653 88.539 426.749 88.539 C 437.998 88.539 445.087 96.398 445.087 107.802 C 445.087 119.205 437.998 127.064 426.749 127.064 C 415.653 127.064 407.948 119.359 407.948 107.802 Z M 498.713 56.332 L 543.402 56.332 L 543.402 40.306 L 498.713 40.306 L 498.713 56.332 Z M 478.526 108.11 C 478.526 132.458 496.402 148.638 521.058 148.638 C 545.56 148.638 563.435 132.458 563.435 108.11 C 563.435 83.762 545.56 67.428 521.058 67.428 C 496.402 67.428 478.526 83.762 478.526 108.11 Z M 502.412 107.956 C 502.412 96.398 509.963 88.693 521.058 88.693 C 531.999 88.693 539.55 96.398 539.55 107.956 C 539.55 119.667 531.999 127.372 521.058 127.372 C 509.963 127.372 502.412 119.667 502.412 107.956 Z" style="fill: rgb(16, 16, 14);"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.7 KiB |
@@ -1,7 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg viewBox="0 0 310 180" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs/>
|
||||
<g transform="matrix(1, 0, 0, 1, -253.876495, 0)">
|
||||
<path d="M 278.049 146.789 L 278.049 127.527 L 287.141 117.972 L 304.4 146.789 L 331.83 146.789 L 303.784 100.251 L 332.755 69.739 L 303.013 69.739 L 278.049 97.477 L 278.049 30.598 L 254.318 30.598 L 254.318 146.789 L 278.049 146.789 Z M 354.169 57.719 C 361.565 57.719 367.575 51.709 367.575 44.158 C 367.575 36.608 361.565 30.752 354.169 30.752 C 346.618 30.752 340.608 36.608 340.608 44.158 C 340.608 51.709 346.618 57.719 354.169 57.719 Z M 342.457 146.789 L 366.188 146.789 L 366.188 69.739 L 342.457 69.739 L 342.457 146.789 Z M 406.407 146.789 L 407.64 136.927 C 411.801 144.015 421.047 148.792 431.834 148.792 C 453.716 148.792 468.972 132.92 468.972 109.035 C 468.972 83.916 455.257 67.119 433.683 67.119 C 422.588 67.119 412.417 71.742 407.794 78.677 L 407.794 30.598 L 384.063 30.598 L 384.063 146.789 L 406.407 146.789 Z M 407.948 107.802 C 407.948 96.244 415.653 88.539 426.749 88.539 C 437.998 88.539 445.087 96.398 445.087 107.802 C 445.087 119.205 437.998 127.064 426.749 127.064 C 415.653 127.064 407.948 119.359 407.948 107.802 Z M 498.713 56.332 L 543.402 56.332 L 543.402 40.306 L 498.713 40.306 L 498.713 56.332 Z M 478.526 108.11 C 478.526 132.458 496.402 148.638 521.058 148.638 C 545.56 148.638 563.435 132.458 563.435 108.11 C 563.435 83.762 545.56 67.428 521.058 67.428 C 496.402 67.428 478.526 83.762 478.526 108.11 Z M 502.412 107.956 C 502.412 96.398 509.963 88.693 521.058 88.693 C 531.999 88.693 539.55 96.398 539.55 107.956 C 539.55 119.667 531.999 127.372 521.058 127.372 C 509.963 127.372 502.412 119.667 502.412 107.956 Z" style="fill: rgb(16, 16, 14);"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 1.8 MiB |
|
Before Width: | Height: | Size: 1.8 MiB |
|
Before Width: | Height: | Size: 496 KiB After Width: | Height: | Size: 496 KiB |
|
Before Width: | Height: | Size: 564 KiB After Width: | Height: | Size: 564 KiB |
|
Before Width: | Height: | Size: 592 KiB After Width: | Height: | Size: 592 KiB |
|
Before Width: | Height: | Size: 453 KiB After Width: | Height: | Size: 453 KiB |
|
Before Width: | Height: | Size: 526 KiB After Width: | Height: | Size: 526 KiB |
@@ -1,19 +0,0 @@
|
||||
[package]
|
||||
name = "biter"
|
||||
description = "A very fast Bitcoin block iterator"
|
||||
version = "0.1.1"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/kibo-money/kibo/tree/main/biter"
|
||||
keywords = ["bitcoin", "block", "iterator"]
|
||||
categories = ["cryptography::cryptocurrencies", "encoding"]
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
bitcoin = { version = "0.32.2", features = ["serde"] }
|
||||
rayon = "1.10.0"
|
||||
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
|
||||
serde = { version = "1.0.204", features = ["derive"] }
|
||||
serde_json = "1.0.122"
|
||||
derived-deref = "2.1.0"
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
# tokio = { version = "1.39.2", features = ["rt-multi-thread"] }
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Biter
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,62 +0,0 @@
|
||||
# Biter
|
||||
|
||||
Biter (Bitcoin Block Iterator) is a very fast and simple Rust library which reads raw block files (*blkXXXXX.dat*) from Bitcoin Core Node and creates an iterator over all the requested blocks in sequential order (0, 1, 2, ...).
|
||||
|
||||
The element returned by the iterator is a tuple which includes the:
|
||||
- Height: `usize`
|
||||
- Block: `Block` (from `bitcoin-rust`)
|
||||
- Block's Hash: `BlockHash` (also from `bitcoin-rust`)
|
||||
|
||||
## Example
|
||||
|
||||
```rust
|
||||
use bitcoincore_rpc::{Auth, Client};
|
||||
|
||||
fn main() {
|
||||
let i = std::time::Instant::now();
|
||||
|
||||
// Path to the Bitcoin data directory
|
||||
let data_dir = "../../bitcoin";
|
||||
|
||||
// Path to the export directory where a mini blk indexer will be exported
|
||||
let export_dir = "./target";
|
||||
|
||||
// Inclusive starting height of the blocks received, `None` for 0
|
||||
let start = Some(850_000);
|
||||
|
||||
// Inclusive ending height of the blocks received, `None` for the last one
|
||||
let end = None;
|
||||
|
||||
// RPC client to filter out forks
|
||||
let url = "http://localhost:8332";
|
||||
let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
let rpc = Client::new(url, auth).unwrap();
|
||||
|
||||
// Create channel receiver then iterate over the blocks
|
||||
biter::new(data_dir, export_dir, start, end, rpc)
|
||||
.iter()
|
||||
.for_each(|(height, _block, hash)| {
|
||||
println!("{height}: {hash}");
|
||||
});
|
||||
|
||||
dbg!(i.elapsed());
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
Even though it reads *blkXXXXX.dat* files, it **needs** `bitcoind` to run with the RPC server to filter out block forks.
|
||||
|
||||
Peak memory should be around 500MB.
|
||||
|
||||
## Comparaison
|
||||
|
||||
| | [biter](https://crates.io/crates/biter) | [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) | [blocks_iterator](https://crates.io/crates/blocks_iterator) |
|
||||
| --- | --- | --- | --- |
|
||||
| Run **with** `bitcoind` | Yes ✅ | No ❌ | Yes ✅ |
|
||||
| Run **without** `bitcoind` | No ❌ | Yes ✅ | Yes ✅ |
|
||||
| `0..=855_000` | 16mn40s | 17mn 46s | > 2h |
|
||||
| `800_000..=855_000` | 2mn 53s (16mn40s if first run) | 3mn 2s | > 2h |
|
||||
|
||||
*Benchmarked on a Macbook Pro M3 Pro*
|
||||
@@ -1,105 +0,0 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::BTreeMap,
|
||||
fs::{self, File},
|
||||
io::{BufReader, BufWriter},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use derived_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{blk_recap::BlkRecap, BlkMetadataAndBlock};
|
||||
|
||||
#[derive(Deref, DerefMut, Debug)]
|
||||
pub struct BlkIndexToBlkRecap {
|
||||
path: String,
|
||||
#[target]
|
||||
tree: BTreeMap<usize, BlkRecap>,
|
||||
}
|
||||
|
||||
impl BlkIndexToBlkRecap {
|
||||
pub fn import(blocks_dir: &BTreeMap<usize, PathBuf>, export_dir: &str) -> Self {
|
||||
let path = format!("{export_dir}/blk_index_to_blk_recap.json");
|
||||
|
||||
let tree = {
|
||||
fs::create_dir_all(export_dir).unwrap();
|
||||
|
||||
if let Ok(file) = File::open(&path) {
|
||||
let reader = BufReader::new(file);
|
||||
serde_json::from_reader(reader).unwrap_or_default()
|
||||
} else {
|
||||
BTreeMap::default()
|
||||
}
|
||||
};
|
||||
|
||||
let mut this = Self { path, tree };
|
||||
|
||||
this.clean_outdated(blocks_dir);
|
||||
|
||||
this
|
||||
}
|
||||
|
||||
pub fn clean_outdated(&mut self, blocks_dir: &BTreeMap<usize, PathBuf>) {
|
||||
blocks_dir.iter().for_each(|(blk_index, blk_path)| {
|
||||
if let Some(blk_recap) = self.get(blk_index) {
|
||||
if blk_recap.has_different_modified_time(blk_path) {
|
||||
self.remove(blk_index);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn get_start_recap(&self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
|
||||
if let Some(start) = start {
|
||||
let (last_key, last_value) = self.last_key_value()?;
|
||||
|
||||
if last_value.height() < start {
|
||||
return Some((*last_key, *last_value));
|
||||
} else if let Some((blk_index, _)) = self
|
||||
.iter()
|
||||
.find(|(_, blk_recap)| blk_recap.is_younger_than(start))
|
||||
{
|
||||
if *blk_index != 0 {
|
||||
let blk_index = *blk_index - 1;
|
||||
return Some((blk_index, *self.get(&blk_index).unwrap()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: usize) {
|
||||
let blk_index = blk_metadata_and_block.blk_metadata.index;
|
||||
|
||||
if let Some(last_entry) = self.last_entry() {
|
||||
// if last_entry.get().is_older_than(height) {
|
||||
match last_entry.key().cmp(&blk_index) {
|
||||
Ordering::Greater => {
|
||||
last_entry.remove_entry();
|
||||
}
|
||||
Ordering::Less => {
|
||||
self.insert(blk_index, BlkRecap::from(height, blk_metadata_and_block));
|
||||
}
|
||||
Ordering::Equal => {}
|
||||
};
|
||||
// }
|
||||
} else {
|
||||
if blk_index != 0 || height != 0 {
|
||||
// dbg!(blk_index, height);
|
||||
unreachable!();
|
||||
}
|
||||
|
||||
self.insert(blk_index, BlkRecap::first(blk_metadata_and_block));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn export(&self) {
|
||||
let file = File::create(&self.path).unwrap_or_else(|_| {
|
||||
dbg!(&self.path);
|
||||
panic!("No such file or directory")
|
||||
});
|
||||
|
||||
serde_json::to_writer_pretty(&mut BufWriter::new(file), &self.tree).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
use bitcoin::Block;
|
||||
|
||||
use crate::BlkMetadata;
|
||||
|
||||
pub struct BlkMetadataAndBlock {
|
||||
pub blk_metadata: BlkMetadata,
|
||||
pub block: Block,
|
||||
}
|
||||
|
||||
impl BlkMetadataAndBlock {
|
||||
pub fn new(blk_metadata: BlkMetadata, block: Block) -> Self {
|
||||
Self {
|
||||
blk_metadata,
|
||||
block,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use bitcoin::{hashes::Hash, BlockHash};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{path_to_modified_time, BlkMetadataAndBlock};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct BlkRecap {
|
||||
min_continuous_height: usize,
|
||||
min_continuous_prev_hash: BlockHash,
|
||||
modified_time: u64,
|
||||
}
|
||||
|
||||
impl BlkRecap {
|
||||
pub fn first(blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
|
||||
Self {
|
||||
min_continuous_height: 0,
|
||||
min_continuous_prev_hash: BlockHash::all_zeros(),
|
||||
modified_time: blk_metadata_and_block.blk_metadata.modified_time,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from(height: usize, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
|
||||
Self {
|
||||
min_continuous_height: height,
|
||||
min_continuous_prev_hash: blk_metadata_and_block.block.header.prev_blockhash,
|
||||
modified_time: blk_metadata_and_block.blk_metadata.modified_time,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn has_different_modified_time(&self, blk_path: &PathBuf) -> bool {
|
||||
self.modified_time != path_to_modified_time(blk_path)
|
||||
}
|
||||
|
||||
pub fn is_younger_than(&self, height: usize) -> bool {
|
||||
self.min_continuous_height > height
|
||||
}
|
||||
|
||||
pub fn height(&self) -> usize {
|
||||
self.min_continuous_height
|
||||
}
|
||||
|
||||
pub fn prev_hash(&self) -> &BlockHash {
|
||||
&self.min_continuous_prev_hash
|
||||
}
|
||||
}
|
||||
@@ -1,384 +0,0 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet, VecDeque},
|
||||
fs::{self},
|
||||
ops::ControlFlow,
|
||||
thread,
|
||||
};
|
||||
|
||||
use bitcoin::{
|
||||
consensus::{Decodable, ReadExt},
|
||||
hashes::Hash,
|
||||
io::{Cursor, Read},
|
||||
Block, BlockHash,
|
||||
};
|
||||
use bitcoincore_rpc::RpcApi;
|
||||
use crossbeam::channel::{bounded, Receiver};
|
||||
use rayon::prelude::*;
|
||||
|
||||
pub use bitcoin;
|
||||
pub use bitcoincore_rpc;
|
||||
|
||||
mod blk_index_to_blk_recap;
|
||||
mod blk_metadata;
|
||||
mod blk_metadata_and_block;
|
||||
mod blk_recap;
|
||||
mod utils;
|
||||
|
||||
use blk_index_to_blk_recap::*;
|
||||
use blk_metadata::*;
|
||||
use blk_metadata_and_block::*;
|
||||
use utils::*;
|
||||
|
||||
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
|
||||
const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
|
||||
const BOUND_CAP: usize = 210;
|
||||
|
||||
enum BlockState {
|
||||
Raw(Vec<u8>),
|
||||
Decoded(Block),
|
||||
}
|
||||
|
||||
///
|
||||
/// Returns a crossbeam channel receiver that receives `(usize, Block, BlockHash)` tuples (with `usize` being the height) in sequential order.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `data_dir` - Path to the Bitcoin data directory
|
||||
/// * `export_dir` - Path to the export directory where a mini blk indexer will be exported
|
||||
/// * `start` - Inclusive starting height of the blocks received, `None` for 0
|
||||
/// * `end` - Inclusive ending height of the blocks received, `None` for the last one
|
||||
/// * `rpc` - RPC client to filter out forks
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use bitcoincore_rpc::{Auth, Client};
|
||||
///
|
||||
/// fn main() {
|
||||
/// let i = std::time::Instant::now();
|
||||
///
|
||||
/// let url = "http://localhost:8332";
|
||||
/// let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
/// let rpc = Client::new(url, auth).unwrap();
|
||||
///
|
||||
/// let data_dir = "../../bitcoin";
|
||||
/// let export_dir = "./target";
|
||||
/// let start = Some(850_000);
|
||||
/// let end = None;
|
||||
///
|
||||
/// biter::new(data_dir, export_dir, start, end, rpc)
|
||||
/// .iter()
|
||||
/// .for_each(|(height, _block, hash)| {
|
||||
/// println!("{height}: {hash}");
|
||||
/// });
|
||||
///
|
||||
/// dbg!(i.elapsed());
|
||||
///}
|
||||
/// ```
|
||||
///
|
||||
pub fn new(
|
||||
data_dir: &str,
|
||||
export_dir: &str,
|
||||
start: Option<usize>,
|
||||
end: Option<usize>,
|
||||
rpc: bitcoincore_rpc::Client,
|
||||
) -> Receiver<(usize, Block, BlockHash)> {
|
||||
let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP);
|
||||
let (send_block, recv_block) = bounded(BOUND_CAP);
|
||||
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
|
||||
|
||||
let blocks_dir = scan_blocks_dir(data_dir);
|
||||
|
||||
let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blocks_dir, export_dir);
|
||||
|
||||
let start_recap = blk_index_to_blk_recap.get_start_recap(start);
|
||||
let starting_blk_index = start_recap.as_ref().map_or(0, |(index, _)| *index);
|
||||
|
||||
thread::spawn(move || {
|
||||
blocks_dir
|
||||
.into_iter()
|
||||
.filter(|(blk_index, _)| blk_index >= &starting_blk_index)
|
||||
.try_for_each(move |(blk_index, blk_path)| {
|
||||
let blk_metadata = BlkMetadata::new(blk_index, &blk_path);
|
||||
|
||||
let blk_bytes = fs::read(&blk_path).unwrap();
|
||||
let blk_bytes_len = blk_bytes.len() as u64;
|
||||
|
||||
let mut cursor = Cursor::new(blk_bytes.as_slice());
|
||||
|
||||
let mut current_4bytes = [0; 4];
|
||||
|
||||
'parent: loop {
|
||||
if cursor.position() == blk_bytes_len {
|
||||
break;
|
||||
}
|
||||
|
||||
// Read until we find a valid suite of MAGIC_BYTES
|
||||
loop {
|
||||
current_4bytes.rotate_left(1);
|
||||
|
||||
if let Ok(byte) = cursor.read_u8() {
|
||||
current_4bytes[3] = byte;
|
||||
} else {
|
||||
break 'parent;
|
||||
}
|
||||
|
||||
if current_4bytes == MAGIC_BYTES {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let block_size = cursor.read_u32().unwrap();
|
||||
|
||||
let mut raw_block = vec![0u8; block_size as usize];
|
||||
|
||||
cursor.read_exact(&mut raw_block).unwrap();
|
||||
|
||||
if send_block_reader
|
||||
.send((blk_metadata, BlockState::Raw(raw_block)))
|
||||
.is_err()
|
||||
{
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
})
|
||||
});
|
||||
|
||||
// thread::spawn(move || {
|
||||
// recv_block_reader.iter().par_bridge().try_for_each(
|
||||
// move |(blk_metadata, mut block_state)| {
|
||||
// let raw_block = match block_state {
|
||||
// BlockState::Raw(vec) => vec,
|
||||
// _ => unreachable!(),
|
||||
// };
|
||||
|
||||
// let mut cursor = Cursor::new(raw_block);
|
||||
|
||||
// block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
|
||||
|
||||
// if send_block
|
||||
// .send(BlkMetadataAndBlock::new(
|
||||
// blk_metadata,
|
||||
// match block_state {
|
||||
// BlockState::Decoded(block) => block,
|
||||
// _ => unreachable!(),
|
||||
// },
|
||||
// ))
|
||||
// .is_err()
|
||||
// {
|
||||
// return ControlFlow::Break(());
|
||||
// }
|
||||
|
||||
// ControlFlow::Continue(())
|
||||
// },
|
||||
// );
|
||||
// });
|
||||
|
||||
// Can't use the previous code because .send() blocks all the threads if full
|
||||
// And other .par_iter() are also stuck because of that
|
||||
thread::spawn(move || {
|
||||
let mut bulk = vec![];
|
||||
|
||||
let drain_and_send = |bulk: &mut Vec<_>| {
|
||||
// Using a vec and sending after to not end up with stuck threads in par iter
|
||||
bulk.par_iter_mut().for_each(|(_, block_state)| {
|
||||
let raw_block = match block_state {
|
||||
BlockState::Raw(vec) => vec,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let mut cursor = Cursor::new(raw_block);
|
||||
|
||||
*block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
|
||||
});
|
||||
|
||||
bulk.drain(..).try_for_each(|(blk_metadata, block_state)| {
|
||||
let block = match block_state {
|
||||
BlockState::Decoded(block) => block,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
if send_block
|
||||
.send(BlkMetadataAndBlock::new(blk_metadata, block))
|
||||
.is_err()
|
||||
{
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
})
|
||||
};
|
||||
|
||||
recv_block_reader.iter().try_for_each(|tuple| {
|
||||
bulk.push(tuple);
|
||||
|
||||
if bulk.len() < BOUND_CAP / 2 {
|
||||
return ControlFlow::Continue(());
|
||||
}
|
||||
|
||||
drain_and_send(&mut bulk)
|
||||
});
|
||||
|
||||
drain_and_send(&mut bulk)
|
||||
});
|
||||
|
||||
// Tokio version: 1022s
|
||||
// Slighlty slower than rayon version
|
||||
// thread::spawn(move || {
|
||||
// let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
// let _guard = rt.enter();
|
||||
|
||||
// let mut tasks = VecDeque::with_capacity(BOUND);
|
||||
|
||||
// recv_block_reader
|
||||
// .iter()
|
||||
// .try_for_each(move |(blk_metadata, block_state)| {
|
||||
// let raw_block = match block_state {
|
||||
// BlockState::Raw(vec) => vec,
|
||||
// _ => unreachable!(),
|
||||
// };
|
||||
|
||||
// tasks.push_back(tokio::task::spawn(async move {
|
||||
// let block = Block::consensus_decode(&mut Cursor::new(raw_block)).unwrap();
|
||||
|
||||
// (blk_metadata, block)
|
||||
// }));
|
||||
|
||||
// while tasks.len() > BOUND {
|
||||
// let (blk_metadata, block) = rt.block_on(tasks.pop_front().unwrap()).unwrap();
|
||||
|
||||
// if send_block
|
||||
// .send(BlkMetadataAndBlock::new(blk_metadata, block))
|
||||
// .is_err()
|
||||
// {
|
||||
// return ControlFlow::Break(());
|
||||
// }
|
||||
// }
|
||||
|
||||
// ControlFlow::Continue(())
|
||||
// });
|
||||
//
|
||||
// todo!("Send the rest")
|
||||
// });
|
||||
|
||||
thread::spawn(move || {
|
||||
let mut height = start_recap.map_or(0, |(_, recap)| recap.height());
|
||||
|
||||
let mut future_blocks = BTreeMap::default();
|
||||
let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default();
|
||||
let mut recent_hashes: BTreeSet<BlockHash> = BTreeSet::default();
|
||||
|
||||
let mut prev_hash =
|
||||
start_recap.map_or_else(BlockHash::all_zeros, |(_, recap)| *recap.prev_hash());
|
||||
|
||||
let mut prepare_and_send = |(hash, tuple): (BlockHash, BlkMetadataAndBlock)| {
|
||||
blk_index_to_blk_recap.update(&tuple, height);
|
||||
|
||||
if start.map_or(true, |start| start <= height) {
|
||||
send_height_block_hash
|
||||
.send((height, tuple.block, hash))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
if end.map_or(false, |end| height == end) {
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
|
||||
height += 1;
|
||||
|
||||
ControlFlow::Continue(())
|
||||
};
|
||||
|
||||
let mut update_tip = |prev_hash: &mut BlockHash,
|
||||
recent_hashes: &mut BTreeSet<BlockHash>,
|
||||
recent_chain: &mut VecDeque<(BlockHash, BlkMetadataAndBlock)>,
|
||||
future_blocks: &mut BTreeMap<BlockHash, BlkMetadataAndBlock>,
|
||||
tuple: BlkMetadataAndBlock| {
|
||||
let mut tuple = Some(tuple);
|
||||
|
||||
while let Some(tuple) = tuple.take().or_else(|| future_blocks.remove(prev_hash)) {
|
||||
let hash = tuple.block.block_hash();
|
||||
|
||||
*prev_hash = hash;
|
||||
recent_hashes.insert(hash);
|
||||
recent_chain.push_back((hash, tuple));
|
||||
}
|
||||
|
||||
while recent_chain.len() > NUMBER_OF_UNSAFE_BLOCKS {
|
||||
let (hash, tuple) = recent_chain.pop_front().unwrap();
|
||||
|
||||
recent_hashes.remove(&hash);
|
||||
|
||||
if prepare_and_send((hash, tuple)).is_break() {
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
};
|
||||
|
||||
let flow = recv_block.iter().try_for_each(|tuple| {
|
||||
// block isn't next after current tip
|
||||
if prev_hash != tuple.block.header.prev_blockhash {
|
||||
let is_block_active =
|
||||
|hash| rpc.get_block_header_info(hash).unwrap().confirmations > 0;
|
||||
|
||||
// block prev has already been processed
|
||||
if recent_hashes.contains(&tuple.block.header.prev_blockhash) {
|
||||
let hash = tuple.block.block_hash();
|
||||
|
||||
if is_block_active(&hash) {
|
||||
let prev_index = recent_chain
|
||||
.iter()
|
||||
.position(|(hash, ..)| hash == &tuple.block.header.prev_blockhash)
|
||||
.unwrap();
|
||||
|
||||
let bad_index_start = prev_index + 1;
|
||||
|
||||
recent_chain.drain(bad_index_start..).for_each(|(hash, _)| {
|
||||
recent_hashes.remove(&hash);
|
||||
});
|
||||
|
||||
return update_tip(
|
||||
&mut prev_hash,
|
||||
&mut recent_hashes,
|
||||
&mut recent_chain,
|
||||
&mut future_blocks,
|
||||
tuple,
|
||||
);
|
||||
}
|
||||
// Check if there was already a future block with the same prev hash
|
||||
} else if let Some(prev_tuple) =
|
||||
future_blocks.insert(tuple.block.header.prev_blockhash, tuple)
|
||||
{
|
||||
// If the previous was the active one
|
||||
if is_block_active(&prev_tuple.block.block_hash()) {
|
||||
// Rollback the insert
|
||||
future_blocks.insert(prev_tuple.block.header.prev_blockhash, prev_tuple);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return update_tip(
|
||||
&mut prev_hash,
|
||||
&mut recent_hashes,
|
||||
&mut recent_chain,
|
||||
&mut future_blocks,
|
||||
tuple,
|
||||
);
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
|
||||
if flow.is_continue() {
|
||||
// Send the last (up to 100) blocks
|
||||
recent_chain.into_iter().try_for_each(prepare_and_send);
|
||||
}
|
||||
|
||||
blk_index_to_blk_recap.export();
|
||||
});
|
||||
|
||||
recv_height_block_hash
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
use bitcoincore_rpc::{Auth, Client};
|
||||
|
||||
fn main() {
|
||||
let i = std::time::Instant::now();
|
||||
|
||||
let url = "http://localhost:8332";
|
||||
let auth = Auth::UserPass("satoshi".to_string(), "nakamoto".to_string());
|
||||
let rpc = Client::new(url, auth).unwrap();
|
||||
|
||||
let data_dir = "../bitcoin";
|
||||
let export_dir = "./target";
|
||||
let start = None;
|
||||
let end = None;
|
||||
|
||||
biter::new(data_dir, export_dir, start, end, rpc)
|
||||
.iter()
|
||||
.for_each(|(height, _block, hash)| {
|
||||
println!("{height}: {hash}");
|
||||
});
|
||||
|
||||
dbg!(i.elapsed());
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
use std::{collections::BTreeMap, fs, path::PathBuf, time::UNIX_EPOCH};
|
||||
|
||||
const BLK: &str = "blk";
|
||||
const DAT: &str = ".dat";
|
||||
|
||||
pub fn scan_blocks_dir(data_dir_path: &str) -> BTreeMap<usize, PathBuf> {
|
||||
let blocks_dir_path = &format!("{data_dir_path}/blocks");
|
||||
|
||||
fs::read_dir(blocks_dir_path)
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.filter(|path| {
|
||||
let is_file = path.is_file();
|
||||
|
||||
if is_file {
|
||||
let file_name = path.file_name().unwrap().to_str().unwrap();
|
||||
|
||||
file_name.starts_with(BLK) && file_name.ends_with(DAT)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
.map(|path| {
|
||||
let file_name = path.file_name().unwrap().to_str().unwrap();
|
||||
|
||||
let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())]
|
||||
.parse::<usize>()
|
||||
.unwrap();
|
||||
|
||||
(blk_index, path)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>()
|
||||
}
|
||||
|
||||
pub fn path_to_modified_time(path: &PathBuf) -> u64 {
|
||||
fs::metadata(path)
|
||||
.unwrap()
|
||||
.modified()
|
||||
.unwrap()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
profile.json.gz
|
||||
flamegraph.svg
|
||||
@@ -0,0 +1,58 @@
|
||||
[package]
|
||||
name = "brk"
|
||||
description.workspace = true
|
||||
license.workspace = true
|
||||
readme.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
edition.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[features]
|
||||
full = [
|
||||
"bundler",
|
||||
"core",
|
||||
"computer",
|
||||
"exit",
|
||||
"fetcher",
|
||||
"indexer",
|
||||
"logger",
|
||||
"mcp",
|
||||
"parser",
|
||||
"interface",
|
||||
"server",
|
||||
"store",
|
||||
"vec",
|
||||
]
|
||||
bundler = ["brk_bundler"]
|
||||
core = ["brk_core"]
|
||||
computer = ["brk_computer"]
|
||||
exit = ["brk_exit"]
|
||||
fetcher = ["brk_fetcher"]
|
||||
indexer = ["brk_indexer"]
|
||||
logger = ["brk_logger"]
|
||||
mcp = ["brk_mcp"]
|
||||
parser = ["brk_parser"]
|
||||
interface = ["brk_interface"]
|
||||
server = ["brk_server"]
|
||||
store = ["brk_store"]
|
||||
vec = ["brk_vec"]
|
||||
|
||||
[dependencies]
|
||||
brk_bundler = { workspace = true, optional = true }
|
||||
brk_cli = { workspace = true }
|
||||
brk_core = { workspace = true, optional = true }
|
||||
brk_computer = { workspace = true, optional = true }
|
||||
brk_exit = { workspace = true, optional = true }
|
||||
brk_fetcher = { workspace = true, optional = true }
|
||||
brk_indexer = { workspace = true, optional = true }
|
||||
brk_logger = { workspace = true, optional = true }
|
||||
brk_mcp = { workspace = true, optional = true }
|
||||
brk_parser = { workspace = true, optional = true }
|
||||
brk_interface = { workspace = true, optional = true }
|
||||
brk_server = { workspace = true, optional = true }
|
||||
brk_store = { workspace = true, optional = true }
|
||||
brk_vec = { workspace = true, optional = true }
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
all-features = true
|
||||
@@ -0,0 +1 @@
|
||||
fn main() {}
|
||||
@@ -0,0 +1,2 @@
|
||||
cargo build --profile profiling
|
||||
flamegraph -- ../../target/profiling/brk
|
||||
@@ -0,0 +1,2 @@
|
||||
cargo build --profile profiling
|
||||
samply record ../../target/profiling/brk
|
||||
@@ -0,0 +1,56 @@
|
||||
#![doc = include_str!(concat!("../", env!("CARGO_PKG_README")))]
|
||||
|
||||
#[cfg(feature = "bundler")]
|
||||
#[doc(inline)]
|
||||
pub use brk_bundler as bundler;
|
||||
|
||||
#[doc(inline)]
|
||||
pub use brk_cli as cli;
|
||||
|
||||
#[cfg(feature = "core")]
|
||||
#[doc(inline)]
|
||||
pub use brk_core as core;
|
||||
|
||||
#[cfg(feature = "computer")]
|
||||
#[doc(inline)]
|
||||
pub use brk_computer as computer;
|
||||
|
||||
#[cfg(feature = "exit")]
|
||||
#[doc(inline)]
|
||||
pub use brk_exit as exit;
|
||||
|
||||
#[cfg(feature = "fetcher")]
|
||||
#[doc(inline)]
|
||||
pub use brk_fetcher as fetcher;
|
||||
|
||||
#[cfg(feature = "indexer")]
|
||||
#[doc(inline)]
|
||||
pub use brk_indexer as indexer;
|
||||
|
||||
#[cfg(feature = "logger")]
|
||||
#[doc(inline)]
|
||||
pub use brk_logger as logger;
|
||||
|
||||
#[cfg(feature = "mcp")]
|
||||
#[doc(inline)]
|
||||
pub use brk_mcp as mcp;
|
||||
|
||||
#[cfg(feature = "parser")]
|
||||
#[doc(inline)]
|
||||
pub use brk_parser as parser;
|
||||
|
||||
#[cfg(feature = "interface")]
|
||||
#[doc(inline)]
|
||||
pub use brk_interface as interface;
|
||||
|
||||
#[cfg(feature = "server")]
|
||||
#[doc(inline)]
|
||||
pub use brk_server as server;
|
||||
|
||||
#[cfg(feature = "store")]
|
||||
#[doc(inline)]
|
||||
pub use brk_store as store;
|
||||
|
||||
#[cfg(feature = "vec")]
|
||||
#[doc(inline)]
|
||||
pub use brk_vec as vec;
|
||||
@@ -0,0 +1 @@
|
||||
use brk_cli::main;
|
||||
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "brk_bundler"
|
||||
description = "A thin wrapper around rolldown"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
log = { workspace = true }
|
||||
notify = "8.1.0"
|
||||
brk_rolldown = "0.1.1"
|
||||
# brk_rolldown = { path = "../../../rolldown/crates/rolldown"}
|
||||
sugar_path = "1.2.0"
|
||||
tokio = { workspace = true }
|
||||
@@ -0,0 +1,145 @@
|
||||
use std::{fs, io, path::Path, sync::Arc};
|
||||
|
||||
use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
|
||||
use log::error;
|
||||
use notify::{EventKind, RecursiveMode, Watcher};
|
||||
use sugar_path::SugarPath;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<()> {
|
||||
let source_path = websites_path.join(source_folder);
|
||||
let dist_path = websites_path.join("dist");
|
||||
|
||||
let _ = fs::remove_dir_all(&dist_path);
|
||||
copy_dir_all(&source_path, &dist_path)?;
|
||||
|
||||
let source_scripts = format!("./{source_folder}/scripts");
|
||||
let source_entry = format!("{source_scripts}/entry.js");
|
||||
|
||||
let absolute_websites_path = websites_path.absolutize();
|
||||
|
||||
let mut bundler = Bundler::new(BundlerOptions {
|
||||
input: Some(vec![source_entry.into()]),
|
||||
dir: Some("./dist/scripts".to_string()),
|
||||
cwd: Some(absolute_websites_path),
|
||||
minify: Some(RawMinifyOptions::Bool(true)),
|
||||
sourcemap: Some(SourceMapType::File),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
bundler.write().await.unwrap();
|
||||
|
||||
let absolute_source_index_path = source_path.join("index.html").absolutize();
|
||||
let absolute_source_index_path_clone = absolute_source_index_path.clone();
|
||||
let absolute_source_path = source_path.absolutize();
|
||||
let absolute_source_path_clone = absolute_source_path.clone();
|
||||
let absolute_source_scripts_path = websites_path.join(source_scripts).absolutize();
|
||||
let absolute_source_sw_path = source_path.join("service-worker.js").absolutize();
|
||||
let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
|
||||
|
||||
let absolute_dist_entry_path = dist_path.join("scripts/entry.js").absolutize();
|
||||
let absolute_dist_index_path = dist_path.join("index.html").absolutize();
|
||||
let absolute_dist_path = dist_path.absolutize();
|
||||
let absolute_dist_path_clone = absolute_dist_path.clone();
|
||||
let absolute_dist_sw_path = dist_path.join("service-worker.js").absolutize();
|
||||
|
||||
let write_index = move || {
|
||||
let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
|
||||
|
||||
if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js")) {
|
||||
if let Some(start) = entry.find("main") {
|
||||
if let Some(end) = entry.find(".js") {
|
||||
let main_hashed = &entry[start..end];
|
||||
contents =
|
||||
contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = fs::write(&absolute_dist_index_path, contents);
|
||||
};
|
||||
|
||||
let write_sw = move || {
|
||||
let contents = fs::read_to_string(&absolute_source_sw_path)
|
||||
.unwrap()
|
||||
.replace("__VERSION__", &format!("v{VERSION}"));
|
||||
let _ = fs::write(&absolute_dist_sw_path, contents);
|
||||
};
|
||||
|
||||
write_index();
|
||||
write_sw();
|
||||
|
||||
if !watch {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let write_index_clone = write_index.clone();
|
||||
|
||||
let mut entry_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(_) => write_index_clone(),
|
||||
Err(e) => error!("watch error: {e:?}"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
entry_watcher
|
||||
.watch(&absolute_dist_entry_path, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let mut source_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(event) => match event.kind {
|
||||
EventKind::Create(_) => event.paths,
|
||||
EventKind::Modify(_) => event.paths,
|
||||
_ => vec![],
|
||||
}
|
||||
.into_iter()
|
||||
.filter(|path| path.starts_with(&absolute_source_path))
|
||||
.filter(|path| !path.starts_with(&absolute_source_scripts_path))
|
||||
.for_each(|source_path| {
|
||||
let suffix = source_path.strip_prefix(&absolute_source_path).unwrap();
|
||||
let dist_path = absolute_dist_path.join(suffix);
|
||||
|
||||
if source_path == absolute_source_index_path_clone {
|
||||
write_index();
|
||||
} else if source_path == absolute_source_sw_path_clone {
|
||||
write_sw();
|
||||
} else {
|
||||
let _ = fs::copy(&source_path, &dist_path);
|
||||
}
|
||||
}),
|
||||
Err(e) => error!("watch error: {e:?}"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
source_watcher
|
||||
.watch(&absolute_source_path_clone, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let watcher =
|
||||
brk_rolldown::Watcher::new(vec![Arc::new(Mutex::new(bundler))], None).unwrap();
|
||||
|
||||
watcher.start().await;
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir_all(&dst)?;
|
||||
for entry in fs::read_dir(src)? {
|
||||
let entry = entry?;
|
||||
let ty = entry.file_type()?;
|
||||
if ty.is_dir() {
|
||||
copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
|
||||
} else {
|
||||
fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "brk_cli"
|
||||
description = "A command line interface to run a Bitcoin Research Kit instance"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
brk_computer = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
brk_fetcher = { workspace = true }
|
||||
brk_indexer = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_server = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
clap_derive = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
log = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
toml = "0.9.0"
|
||||
|
||||
[[bin]]
|
||||
name = "brk"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.dist]
|
||||
dist = false
|
||||
@@ -0,0 +1,96 @@
|
||||
# BRK CLI
|
||||
|
||||
<p align="left">
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
<a href="https://crates.io/crates/brk_cli">
|
||||
<img src="https://img.shields.io/crates/v/brk_cli" alt="Version" />
|
||||
</a>
|
||||
<a href="https://docs.rs/brk_cli">
|
||||
<img src="https://img.shields.io/docsrs/brk_cli" alt="Documentation" />
|
||||
</a>
|
||||
<img src="https://img.shields.io/crates/size/brk_cli" alt="Size" />
|
||||
<a href="https://deps.rs/crate/brk_cli">
|
||||
<img src="https://deps.rs/crate/brk_cli/latest/status.svg" alt="Dependency status">
|
||||
</a>
|
||||
<a href="https://discord.gg/HaR3wpH3nr">
|
||||
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
|
||||
</a>
|
||||
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
|
||||
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
|
||||
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
|
||||
</a>
|
||||
<a href="https://x.com/brkdotorg">
|
||||
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
|
||||
</a>
|
||||
</p>
|
||||
|
||||
A command line interface to run a Bitcoin Research Kit instance.
|
||||
|
||||
It's very customizable with all parameters from the underlying tools (crates) used inside.
|
||||
|
||||
Run `brk -h` for more information.
|
||||
|
||||
## Requirements
|
||||
|
||||
### Hardware
|
||||
|
||||
#### Recommended
|
||||
|
||||
- [Latest base model Mac mini](https://www.apple.com/mac-mini/)
|
||||
- [Thunderbolt 4 SSD enclosure](https://satechi.net/products/usb4-nvme-ssd-pro-enclosure/Z2lkOi8vc2hvcGlmeS9Qcm9kdWN0VmFyaWFudC80MDE4ODQ3MDA2NzI4OA==?queryID=7961465089021ee203a60db7e62e90d2)
|
||||
- [2 TB NVMe SSD](https://shop.sandisk.com/products/ssd/internal-ssd/wd-black-sn850x-nvme-ssd?sku=WDS200T2X0E-00BCA0)
|
||||
|
||||
#### Minimum
|
||||
|
||||
To be determined
|
||||
|
||||
### Software
|
||||
|
||||
- [Bitcoin](https://bitcoin.org/en/full-node)
|
||||
- [Rust](https://www.rust-lang.org/tools/install)
|
||||
- Unix based operating system (Mac OS or Linux)
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config`
|
||||
|
||||
## Download
|
||||
|
||||
### Binaries
|
||||
|
||||
You can find a pre-built binary for your operating system in the [releases page](https://github.com/bitcoinresearchkit/brk/releases/latest).
|
||||
|
||||
### Cargo
|
||||
|
||||
```bash
|
||||
# Install
|
||||
cargo install brk # or `cargo install brk_cli`, the result is the same
|
||||
|
||||
# Update
|
||||
cargo install brk # or `cargo install-update -a` if you have `cargo-update` installed
|
||||
```
|
||||
|
||||
### Source
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bitcoinresearchkit/brk.git
|
||||
cd brk/crates/brk
|
||||
cargo run -r
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Run `brk -h` to view each available parameter and their respective description.
|
||||
|
||||
> [!TIP]
|
||||
> Every parameter set will be saved at `~/.brk/config.toml`, which allows you to simply run `brk` next time.
|
||||
|
||||
## Tunnel
|
||||
|
||||
The easiest way to let others access your server is to use `cloudflared` which will also cache requests. For more information see [Cloudflare Tunnel](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) documentation.
|
||||
@@ -0,0 +1,376 @@
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use bitcoincore_rpc::{self, Auth, Client};
|
||||
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_server::Website;
|
||||
use brk_vec::{Computation, Format};
|
||||
use clap::Parser;
|
||||
use clap_derive::Parser;
|
||||
use color_eyre::eyre::eyre;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::services::Services;
|
||||
|
||||
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
|
||||
#[command(version, about)]
|
||||
pub struct Config {
|
||||
/// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
bitcoindir: Option<String>,
|
||||
|
||||
/// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
blocksdir: Option<String>,
|
||||
|
||||
/// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
brkdir: Option<String>,
|
||||
|
||||
/// Activated services, default: all, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
services: Option<Services>,
|
||||
|
||||
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
computation: Option<Computation>,
|
||||
|
||||
/// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
format: Option<Format>,
|
||||
|
||||
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short = 'F', long, value_name = "BOOL")]
|
||||
fetch: Option<bool>,
|
||||
|
||||
/// Website served by the server (if active), default: default, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
website: Option<Website>,
|
||||
|
||||
/// Bitcoin RPC ip, default: localhost, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "IP")]
|
||||
rpcconnect: Option<String>,
|
||||
|
||||
/// Bitcoin RPC port, default: 8332, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PORT")]
|
||||
rpcport: Option<u16>,
|
||||
|
||||
/// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
rpccookiefile: Option<String>,
|
||||
|
||||
/// Bitcoin RPC username, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "USERNAME")]
|
||||
rpcuser: Option<String>,
|
||||
|
||||
/// Bitcoin RPC password, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PASSWORD")]
|
||||
rpcpassword: Option<String>,
|
||||
|
||||
/// Delay between runs, default: 0, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "SECONDS")]
|
||||
delay: Option<u64>,
|
||||
|
||||
/// Activate the Model Context Protocol (MCP) endpoint to give LLMs access to BRK (experimental), default: true, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
mcp: Option<bool>,
|
||||
|
||||
/// DEV: Activate watching the selected website's folder for changes, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
watch: Option<bool>,
|
||||
|
||||
/// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
check_collisions: Option<bool>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn import() -> color_eyre::Result<Self> {
|
||||
let config_args = Some(Config::parse());
|
||||
|
||||
let path = dot_brk_path();
|
||||
|
||||
let _ = fs::create_dir_all(&path);
|
||||
|
||||
let path = path.join("config.toml");
|
||||
|
||||
let mut config_saved = Self::read(&path);
|
||||
|
||||
if let Some(mut config_args) = config_args {
|
||||
if let Some(bitcoindir) = config_args.bitcoindir.take() {
|
||||
config_saved.bitcoindir = Some(bitcoindir);
|
||||
}
|
||||
|
||||
if let Some(blocksdir) = config_args.blocksdir.take() {
|
||||
config_saved.blocksdir = Some(blocksdir);
|
||||
}
|
||||
|
||||
if let Some(brkdir) = config_args.brkdir.take() {
|
||||
config_saved.brkdir = Some(brkdir);
|
||||
}
|
||||
|
||||
if let Some(services) = config_args.services.take() {
|
||||
config_saved.services = Some(services);
|
||||
}
|
||||
|
||||
if let Some(computation) = config_args.computation.take() {
|
||||
config_saved.computation = Some(computation);
|
||||
}
|
||||
|
||||
if let Some(fetch) = config_args.fetch.take() {
|
||||
config_saved.fetch = Some(fetch);
|
||||
}
|
||||
|
||||
if let Some(format) = config_args.format.take() {
|
||||
config_saved.format = Some(format);
|
||||
}
|
||||
|
||||
if let Some(website) = config_args.website.take() {
|
||||
config_saved.website = Some(website);
|
||||
}
|
||||
|
||||
if let Some(rpcconnect) = config_args.rpcconnect.take() {
|
||||
config_saved.rpcconnect = Some(rpcconnect);
|
||||
}
|
||||
|
||||
if let Some(rpcport) = config_args.rpcport.take() {
|
||||
config_saved.rpcport = Some(rpcport);
|
||||
}
|
||||
|
||||
if let Some(rpccookiefile) = config_args.rpccookiefile.take() {
|
||||
config_saved.rpccookiefile = Some(rpccookiefile);
|
||||
}
|
||||
|
||||
if let Some(rpcuser) = config_args.rpcuser.take() {
|
||||
config_saved.rpcuser = Some(rpcuser);
|
||||
}
|
||||
|
||||
if let Some(rpcpassword) = config_args.rpcpassword.take() {
|
||||
config_saved.rpcpassword = Some(rpcpassword);
|
||||
}
|
||||
|
||||
if let Some(delay) = config_args.delay.take() {
|
||||
config_saved.delay = Some(delay);
|
||||
}
|
||||
|
||||
if let Some(check_collisions) = config_args.check_collisions.take() {
|
||||
config_saved.check_collisions = Some(check_collisions);
|
||||
}
|
||||
|
||||
if let Some(mcp) = config_args.mcp.take() {
|
||||
config_saved.mcp = Some(mcp);
|
||||
}
|
||||
|
||||
if let Some(watch) = config_args.watch.take() {
|
||||
config_saved.watch = Some(watch);
|
||||
}
|
||||
|
||||
if config_args != Config::default() {
|
||||
dbg!(config_args);
|
||||
panic!("Didn't consume the full config")
|
||||
}
|
||||
}
|
||||
|
||||
let config = config_saved;
|
||||
|
||||
config.check();
|
||||
|
||||
config.write(&path)?;
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn check(&self) {
|
||||
if !self.bitcoindir().is_dir() {
|
||||
println!("{:?} isn't a valid directory", self.bitcoindir());
|
||||
println!("Please use the --bitcoindir parameter to set a valid path.");
|
||||
println!("Run the program with '-h' for help.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
if !self.blocksdir().is_dir() {
|
||||
println!("{:?} isn't a valid directory", self.blocksdir());
|
||||
println!("Please use the --blocksdir parameter to set a valid path.");
|
||||
println!("Run the program with '-h' for help.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
if !self.brkdir().is_dir() {
|
||||
println!("{:?} isn't a valid directory", self.brkdir());
|
||||
println!("Please use the --brkdir parameter to set a valid path.");
|
||||
println!("Run the program with '-h' for help.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
if self.rpc_auth().is_err() {
|
||||
println!(
|
||||
"No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help."
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn read(path: &Path) -> Self {
|
||||
fs::read_to_string(path).map_or_else(
|
||||
|_| Config::default(),
|
||||
|contents| toml::from_str(&contents).unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
fn write(&self, path: &Path) -> std::io::Result<()> {
|
||||
fs::write(path, toml::to_string(self).unwrap())
|
||||
}
|
||||
|
||||
pub fn rpc(&self) -> color_eyre::Result<&'static Client> {
|
||||
Ok(Box::leak(Box::new(Client::new(
|
||||
&format!(
|
||||
"http://{}:{}",
|
||||
self.rpcconnect().unwrap_or(&"localhost".to_string()),
|
||||
self.rpcport().unwrap_or(8332)
|
||||
),
|
||||
self.rpc_auth().unwrap(),
|
||||
)?)))
|
||||
}
|
||||
|
||||
fn rpc_auth(&self) -> color_eyre::Result<Auth> {
|
||||
let cookie = self.path_cookiefile();
|
||||
|
||||
if cookie.is_file() {
|
||||
Ok(Auth::CookieFile(cookie))
|
||||
} else if self.rpcuser.is_some() && self.rpcpassword.is_some() {
|
||||
Ok(Auth::UserPass(
|
||||
self.rpcuser.clone().unwrap(),
|
||||
self.rpcpassword.clone().unwrap(),
|
||||
))
|
||||
} else {
|
||||
Err(eyre!("Failed to find correct auth"))
|
||||
}
|
||||
}
|
||||
|
||||
fn rpcconnect(&self) -> Option<&String> {
|
||||
self.rpcconnect.as_ref()
|
||||
}
|
||||
|
||||
fn rpcport(&self) -> Option<u16> {
|
||||
self.rpcport
|
||||
}
|
||||
|
||||
pub fn delay(&self) -> Option<u64> {
|
||||
self.delay
|
||||
}
|
||||
|
||||
pub fn bitcoindir(&self) -> PathBuf {
|
||||
self.bitcoindir
|
||||
.as_ref()
|
||||
.map_or_else(default_bitcoin_path, |s| Self::fix_user_path(s.as_ref()))
|
||||
}
|
||||
|
||||
pub fn blocksdir(&self) -> PathBuf {
|
||||
self.blocksdir.as_ref().map_or_else(
|
||||
|| self.bitcoindir().join("blocks"),
|
||||
|blocksdir| Self::fix_user_path(blocksdir.as_str()),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn brkdir(&self) -> PathBuf {
|
||||
self.brkdir
|
||||
.as_ref()
|
||||
.map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref()))
|
||||
}
|
||||
|
||||
pub fn outputsdir(&self) -> PathBuf {
|
||||
self.brkdir().join("outputs")
|
||||
}
|
||||
|
||||
pub fn harsdir(&self) -> PathBuf {
|
||||
self.outputsdir().join("hars")
|
||||
}
|
||||
|
||||
pub fn process(&self) -> bool {
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Processor)
|
||||
}
|
||||
|
||||
pub fn serve(&self) -> bool {
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Server)
|
||||
}
|
||||
|
||||
fn path_cookiefile(&self) -> PathBuf {
|
||||
self.rpccookiefile.as_ref().map_or_else(
|
||||
|| self.bitcoindir().join(".cookie"),
|
||||
|p| Self::fix_user_path(p.as_str()),
|
||||
)
|
||||
}
|
||||
|
||||
fn fix_user_path(path: &str) -> PathBuf {
|
||||
let fix = move |pattern: &str| {
|
||||
if path.starts_with(pattern) {
|
||||
let path = &path
|
||||
.replace(&format!("{pattern}/"), "")
|
||||
.replace(pattern, "");
|
||||
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
|
||||
Some(Path::new(&home).join(path))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
fix("~").unwrap_or_else(|| fix("$HOME").unwrap_or_else(|| PathBuf::from(&path)))
|
||||
}
|
||||
|
||||
pub fn website(&self) -> Website {
|
||||
self.website.unwrap_or(Website::Default)
|
||||
}
|
||||
|
||||
pub fn fetch(&self) -> bool {
|
||||
self.fetch.is_none_or(|b| b)
|
||||
}
|
||||
|
||||
pub fn fetcher(&self) -> Option<Fetcher> {
|
||||
self.fetch()
|
||||
.then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap())
|
||||
}
|
||||
|
||||
pub fn computation(&self) -> Computation {
|
||||
self.computation.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn format(&self) -> Format {
|
||||
self.format.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn check_collisions(&self) -> bool {
|
||||
self.check_collisions.is_some_and(|b| b)
|
||||
}
|
||||
|
||||
pub fn mcp(&self) -> bool {
|
||||
self.mcp.is_none_or(|b| b)
|
||||
}
|
||||
|
||||
pub fn watch(&self) -> bool {
|
||||
self.watch.is_some_and(|b| b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
use std::{fs, thread};
|
||||
|
||||
use brk_core::{dot_brk_log_path, dot_brk_path};
|
||||
|
||||
mod config;
|
||||
mod run;
|
||||
mod services;
|
||||
|
||||
use run::*;
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
fs::create_dir_all(dot_brk_path())?;
|
||||
|
||||
brk_logger::init(Some(&dot_brk_log_path()));
|
||||
|
||||
thread::Builder::new()
|
||||
.stack_size(256 * 1024 * 1024)
|
||||
.spawn(run)?
|
||||
.join()
|
||||
.unwrap()
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
use brk_cli::main;
|
||||
@@ -0,0 +1,103 @@
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use bitcoincore_rpc::{self, RpcApi};
|
||||
use brk_computer::Computer;
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_server::Server;
|
||||
use log::info;
|
||||
|
||||
use crate::config::Config;
|
||||
|
||||
pub fn run() -> color_eyre::Result<()> {
|
||||
let config = Config::import()?;
|
||||
|
||||
let rpc = config.rpc()?;
|
||||
|
||||
let exit = Exit::new();
|
||||
|
||||
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
|
||||
|
||||
let format = config.format();
|
||||
|
||||
let mut indexer = Indexer::forced_import(&config.outputsdir())?;
|
||||
|
||||
let wait_for_synced_node = || -> color_eyre::Result<()> {
|
||||
let is_synced = || -> color_eyre::Result<bool> {
|
||||
let info = rpc.get_blockchain_info()?;
|
||||
Ok(info.headers == info.blocks)
|
||||
};
|
||||
|
||||
if !is_synced()? {
|
||||
info!("Waiting for node to be synced...");
|
||||
while !is_synced()? {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let mut computer = Computer::forced_import(
|
||||
&config.outputsdir(),
|
||||
&indexer,
|
||||
config.computation(),
|
||||
config.fetcher(),
|
||||
format,
|
||||
)?;
|
||||
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?
|
||||
.block_on(async {
|
||||
let server = if config.serve() {
|
||||
let served_indexer = indexer.clone();
|
||||
let served_computer = computer.clone();
|
||||
|
||||
let server = Server::new(served_indexer, served_computer, config.website())?;
|
||||
|
||||
let watch = config.watch();
|
||||
let mcp = config.mcp();
|
||||
let opt = Some(tokio::spawn(async move {
|
||||
server.serve(watch, mcp).await.unwrap();
|
||||
}));
|
||||
|
||||
sleep(Duration::from_secs(1));
|
||||
|
||||
opt
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if config.process() {
|
||||
loop {
|
||||
wait_for_synced_node()?;
|
||||
|
||||
let block_count = rpc.get_block_count()?;
|
||||
|
||||
info!("{} blocks found.", block_count + 1);
|
||||
|
||||
let starting_indexes =
|
||||
indexer.index(&parser, rpc, &exit, config.check_collisions())?;
|
||||
|
||||
computer.compute(&mut indexer, starting_indexes, &exit)?;
|
||||
|
||||
if let Some(delay) = config.delay() {
|
||||
sleep(Duration::from_secs(delay))
|
||||
}
|
||||
|
||||
info!("Waiting for new blocks...");
|
||||
|
||||
while block_count == rpc.get_block_count()? {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(handle) = server {
|
||||
handle.await.unwrap();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
use clap_derive::{Parser, ValueEnum};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
Default,
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
Parser,
|
||||
ValueEnum,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
)]
|
||||
pub enum Services {
|
||||
#[default]
|
||||
All,
|
||||
Processor,
|
||||
Server,
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "brk_computer"
|
||||
description = "A Bitcoin dataset computer, built on top of brk_indexer"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
bincode = { workspace = true }
|
||||
bitcoin = { workspace = true }
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
brk_fetcher = { workspace = true }
|
||||
brk_indexer = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_store = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
derive_deref = { workspace = true }
|
||||
either = "1.15.0"
|
||||
fjall = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
zerocopy = { workspace = true }
|
||||
zerocopy-derive = { workspace = true }
|
||||
|
||||
[package.metadata.cargo-machete]
|
||||
ignored = ["zerocopy"]
|
||||
@@ -0,0 +1,34 @@
|
||||
# BRK Computer
|
||||
|
||||
<p align="left">
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
<a href="https://crates.io/crates/brk_computer">
|
||||
<img src="https://img.shields.io/crates/v/brk_computer" alt="Version" />
|
||||
</a>
|
||||
<a href="https://docs.rs/brk_computer">
|
||||
<img src="https://img.shields.io/docsrs/brk_computer" alt="Documentation" />
|
||||
</a>
|
||||
<img src="https://img.shields.io/crates/size/brk_computer" alt="Size" />
|
||||
<a href="https://deps.rs/crate/brk_computer">
|
||||
<img src="https://deps.rs/crate/brk_computer/latest/status.svg" alt="Dependency status">
|
||||
</a>
|
||||
<a href="https://discord.gg/HaR3wpH3nr">
|
||||
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
|
||||
</a>
|
||||
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
|
||||
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
|
||||
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
|
||||
</a>
|
||||
<a href="https://x.com/brkdotorg">
|
||||
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
|
||||
</a>
|
||||
</p>
|
||||
|
||||
A dataset computer, built on top of `brk_indexer` and `brk_fetcher`. It computes any dataset you can think of and if it doesn't feel free to create an issue.
|
||||
@@ -0,0 +1,56 @@
|
||||
use std::{path::Path, thread};
|
||||
|
||||
use brk_computer::Computer;
|
||||
use brk_core::{default_bitcoin_path, default_brk_path};
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::Parser;
|
||||
use brk_vec::{Computation, Format};
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
brk_logger::init(Some(Path::new(".log")));
|
||||
|
||||
let bitcoin_dir = default_bitcoin_path();
|
||||
|
||||
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
|
||||
"http://localhost:8332",
|
||||
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
|
||||
)?));
|
||||
let exit = Exit::new();
|
||||
|
||||
// Can't increase main thread's stack programatically, thus we need to use another thread
|
||||
thread::Builder::new()
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(move || -> color_eyre::Result<()> {
|
||||
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
|
||||
|
||||
let _outputs_dir = default_brk_path().join("outputs");
|
||||
let outputs_dir = _outputs_dir.as_path();
|
||||
// let outputs_dir = Path::new("../../_outputs");
|
||||
|
||||
let format = Format::Raw;
|
||||
|
||||
let mut indexer = Indexer::forced_import(outputs_dir)?;
|
||||
|
||||
let fetcher = Fetcher::import(None)?;
|
||||
|
||||
let mut computer = Computer::forced_import(
|
||||
outputs_dir,
|
||||
&indexer,
|
||||
Computation::Lazy,
|
||||
Some(fetcher),
|
||||
format,
|
||||
)?;
|
||||
|
||||
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
|
||||
|
||||
computer.compute(&mut indexer, starting_indexes, &exit)?;
|
||||
|
||||
Ok(())
|
||||
})?
|
||||
.join()
|
||||
.unwrap()
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
#![doc = include_str!("../README.md")]
|
||||
#![doc = "\n## Example\n\n```rust"]
|
||||
#![doc = include_str!("../examples/main.rs")]
|
||||
#![doc = "```"]
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::Version;
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{Computation, Format};
|
||||
use log::info;
|
||||
|
||||
mod states;
|
||||
mod stores;
|
||||
mod utils;
|
||||
mod vecs;
|
||||
|
||||
use states::*;
|
||||
use stores::Stores;
|
||||
use vecs::Vecs;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Computer {
|
||||
fetcher: Option<Fetcher>,
|
||||
pub vecs: Vecs,
|
||||
pub stores: Stores,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ONE;
|
||||
|
||||
impl Computer {
|
||||
/// Do NOT import multiple times or things will break !!!
|
||||
pub fn forced_import(
|
||||
outputs_dir: &Path,
|
||||
indexer: &Indexer,
|
||||
computation: Computation,
|
||||
fetcher: Option<Fetcher>,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
vecs: Vecs::import(
|
||||
// TODO: Give self.path, join inside import
|
||||
&outputs_dir.join("vecs/computed"),
|
||||
VERSION + Version::ZERO,
|
||||
indexer,
|
||||
fetcher.is_some(),
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
stores: Stores::import(
|
||||
// TODO: Give self.path, join inside import
|
||||
&outputs_dir.join("stores"),
|
||||
VERSION + Version::ZERO,
|
||||
&indexer.stores.keyspace,
|
||||
)?,
|
||||
fetcher,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Computer {
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexer: &mut Indexer,
|
||||
starting_indexes: brk_indexer::Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
info!("Computing...");
|
||||
self.vecs.compute(
|
||||
indexer,
|
||||
starting_indexes,
|
||||
self.fetcher.as_mut(),
|
||||
exit,
|
||||
&mut self.stores,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
use std::ops::{Add, AddAssign, SubAssign};
|
||||
|
||||
use brk_core::{Dollars, Timestamp};
|
||||
|
||||
use super::SupplyState;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlockState {
|
||||
pub supply: SupplyState,
|
||||
pub price: Option<Dollars>,
|
||||
pub timestamp: Timestamp,
|
||||
}
|
||||
|
||||
impl Add<BlockState> for BlockState {
|
||||
type Output = Self;
|
||||
fn add(mut self, rhs: BlockState) -> Self::Output {
|
||||
self.supply += &rhs.supply;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl AddAssign<&BlockState> for BlockState {
|
||||
fn add_assign(&mut self, rhs: &Self) {
|
||||
self.supply += &rhs.supply;
|
||||
}
|
||||
}
|
||||
|
||||
impl SubAssign<&BlockState> for BlockState {
|
||||
fn sub_assign(&mut self, rhs: &Self) {
|
||||
self.supply -= &rhs.supply;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{AddressData, Dollars, Height, Result, Sats};
|
||||
|
||||
use crate::SupplyState;
|
||||
|
||||
use super::CohortState;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AddressCohortState {
|
||||
pub address_count: usize,
|
||||
pub inner: CohortState,
|
||||
}
|
||||
|
||||
impl AddressCohortState {
|
||||
pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result<Self> {
|
||||
Ok(Self {
|
||||
address_count: 0,
|
||||
inner: CohortState::default_and_import(path, name, compute_dollars)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn height(&self) -> Option<Height> {
|
||||
self.inner.height()
|
||||
}
|
||||
|
||||
pub fn reset_price_to_amount(&mut self) -> Result<()> {
|
||||
self.inner.reset_price_to_amount()
|
||||
}
|
||||
|
||||
pub fn reset_single_iteration_values(&mut self) {
|
||||
self.inner.reset_single_iteration_values();
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn send(
|
||||
&mut self,
|
||||
addressdata: &mut AddressData,
|
||||
value: Sats,
|
||||
current_price: Option<Dollars>,
|
||||
prev_price: Option<Dollars>,
|
||||
blocks_old: usize,
|
||||
days_old: f64,
|
||||
older_than_hour: bool,
|
||||
) -> Result<()> {
|
||||
let compute_price = current_price.is_some();
|
||||
|
||||
let prev_realized_price = compute_price.then(|| addressdata.realized_price());
|
||||
let prev_supply_state = SupplyState {
|
||||
utxos: addressdata.outputs_len as usize,
|
||||
value: addressdata.amount(),
|
||||
};
|
||||
|
||||
addressdata.send(value, prev_price)?;
|
||||
|
||||
let supply_state = SupplyState {
|
||||
utxos: addressdata.outputs_len as usize,
|
||||
value: addressdata.amount(),
|
||||
};
|
||||
|
||||
self.inner.send_(
|
||||
&SupplyState { utxos: 1, value },
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old,
|
||||
older_than_hour,
|
||||
compute_price.then(|| (addressdata.realized_price(), &supply_state)),
|
||||
prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option<Dollars>) {
|
||||
let compute_price = price.is_some();
|
||||
|
||||
let prev_realized_price = compute_price.then(|| address_data.realized_price());
|
||||
let prev_supply_state = SupplyState {
|
||||
utxos: address_data.outputs_len as usize,
|
||||
value: address_data.amount(),
|
||||
};
|
||||
|
||||
address_data.receive(value, price);
|
||||
|
||||
let supply_state = SupplyState {
|
||||
utxos: address_data.outputs_len as usize,
|
||||
value: address_data.amount(),
|
||||
};
|
||||
|
||||
self.inner.receive_(
|
||||
&SupplyState { utxos: 1, value },
|
||||
price,
|
||||
compute_price.then(|| (address_data.realized_price(), &supply_state)),
|
||||
prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)),
|
||||
);
|
||||
}
|
||||
|
||||
pub fn add(&mut self, addressdata: &AddressData) {
|
||||
self.address_count += 1;
|
||||
self.inner.increment_(
|
||||
&addressdata.into(),
|
||||
addressdata.realized_cap,
|
||||
addressdata.realized_price(),
|
||||
);
|
||||
}
|
||||
|
||||
pub fn subtract(&mut self, addressdata: &AddressData) {
|
||||
self.address_count = self.address_count.checked_sub(1).unwrap();
|
||||
self.inner.decrement_(
|
||||
&addressdata.into(),
|
||||
addressdata.realized_cap,
|
||||
addressdata.realized_price(),
|
||||
);
|
||||
}
|
||||
|
||||
pub fn commit(&mut self, height: Height) -> Result<()> {
|
||||
self.inner.commit(height)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
use std::{cmp::Ordering, path::Path};
|
||||
|
||||
use brk_core::{CheckedSub, Dollars, Height, Result, Sats};
|
||||
|
||||
use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CohortState {
|
||||
pub supply: SupplyState,
|
||||
pub realized: Option<RealizedState>,
|
||||
pub satblocks_destroyed: Sats,
|
||||
pub satdays_destroyed: Sats,
|
||||
|
||||
price_to_amount: PriceToAmount,
|
||||
}
|
||||
|
||||
impl CohortState {
|
||||
pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result<Self> {
|
||||
Ok(Self {
|
||||
supply: SupplyState::default(),
|
||||
realized: compute_dollars.then_some(RealizedState::NAN),
|
||||
satblocks_destroyed: Sats::ZERO,
|
||||
satdays_destroyed: Sats::ZERO,
|
||||
price_to_amount: PriceToAmount::forced_import(path, name),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn height(&self) -> Option<Height> {
|
||||
self.price_to_amount.height()
|
||||
}
|
||||
|
||||
pub fn reset_price_to_amount(&mut self) -> Result<()> {
|
||||
self.price_to_amount.reset()
|
||||
}
|
||||
|
||||
pub fn price_to_amount_first_key_value(&self) -> Option<(&Dollars, &Sats)> {
|
||||
self.price_to_amount.first_key_value()
|
||||
}
|
||||
|
||||
pub fn price_to_amount_last_key_value(&self) -> Option<(&Dollars, &Sats)> {
|
||||
self.price_to_amount.last_key_value()
|
||||
}
|
||||
|
||||
pub fn reset_single_iteration_values(&mut self) {
|
||||
self.satdays_destroyed = Sats::ZERO;
|
||||
self.satblocks_destroyed = Sats::ZERO;
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.reset_single_iteration_values();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn increment(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
|
||||
self.supply += supply_state;
|
||||
|
||||
if supply_state.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let price = price.unwrap();
|
||||
realized.increment(supply_state, price);
|
||||
self.price_to_amount.increment(price, supply_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn increment_(
|
||||
&mut self,
|
||||
supply_state: &SupplyState,
|
||||
realized_cap: Dollars,
|
||||
realized_price: Dollars,
|
||||
) {
|
||||
self.supply += supply_state;
|
||||
|
||||
if supply_state.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.increment_(realized_cap);
|
||||
self.price_to_amount.increment(realized_price, supply_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrement(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
|
||||
self.supply -= supply_state;
|
||||
|
||||
if supply_state.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let price = price.unwrap();
|
||||
realized.decrement(supply_state, price);
|
||||
self.price_to_amount.decrement(price, supply_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrement_(
|
||||
&mut self,
|
||||
supply_state: &SupplyState,
|
||||
realized_cap: Dollars,
|
||||
realized_price: Dollars,
|
||||
) {
|
||||
self.supply -= supply_state;
|
||||
|
||||
if supply_state.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.decrement_(realized_cap);
|
||||
self.price_to_amount.decrement(realized_price, supply_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
|
||||
self.receive_(
|
||||
supply_state,
|
||||
price,
|
||||
price.map(|price| (price, supply_state)),
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn receive_(
|
||||
&mut self,
|
||||
supply_state: &SupplyState,
|
||||
price: Option<Dollars>,
|
||||
price_to_amount_increment: Option<(Dollars, &SupplyState)>,
|
||||
price_to_amount_decrement: Option<(Dollars, &SupplyState)>,
|
||||
) {
|
||||
self.supply += supply_state;
|
||||
|
||||
if supply_state.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let price = price.unwrap();
|
||||
realized.receive(supply_state, price);
|
||||
|
||||
if let Some((price, supply)) = price_to_amount_increment
|
||||
&& supply.value.is_not_zero()
|
||||
{
|
||||
self.price_to_amount.increment(price, supply);
|
||||
}
|
||||
if let Some((price, supply)) = price_to_amount_decrement
|
||||
&& supply.value.is_not_zero()
|
||||
{
|
||||
self.price_to_amount.decrement(price, supply);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send(
|
||||
&mut self,
|
||||
supply_state: &SupplyState,
|
||||
current_price: Option<Dollars>,
|
||||
prev_price: Option<Dollars>,
|
||||
blocks_old: usize,
|
||||
days_old: f64,
|
||||
older_than_hour: bool,
|
||||
) {
|
||||
self.send_(
|
||||
supply_state,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old,
|
||||
older_than_hour,
|
||||
None,
|
||||
prev_price.map(|prev_price| (prev_price, supply_state)),
|
||||
);
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn send_(
|
||||
&mut self,
|
||||
supply_state: &SupplyState,
|
||||
current_price: Option<Dollars>,
|
||||
prev_price: Option<Dollars>,
|
||||
blocks_old: usize,
|
||||
days_old: f64,
|
||||
older_than_hour: bool,
|
||||
price_to_amount_increment: Option<(Dollars, &SupplyState)>,
|
||||
price_to_amount_decrement: Option<(Dollars, &SupplyState)>,
|
||||
) {
|
||||
if supply_state.utxos == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
self.supply -= supply_state;
|
||||
|
||||
if supply_state.value > Sats::ZERO {
|
||||
self.satblocks_destroyed += supply_state.value * blocks_old;
|
||||
|
||||
self.satdays_destroyed +=
|
||||
Sats::from((u64::from(supply_state.value) as f64 * days_old).floor() as u64);
|
||||
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let current_price = current_price.unwrap();
|
||||
let prev_price = prev_price.unwrap();
|
||||
realized.send(supply_state, current_price, prev_price, older_than_hour);
|
||||
if let Some((price, supply)) = price_to_amount_increment
|
||||
&& supply.value.is_not_zero()
|
||||
{
|
||||
self.price_to_amount.increment(price, supply);
|
||||
}
|
||||
if let Some((price, supply)) = price_to_amount_decrement
|
||||
&& supply.value.is_not_zero()
|
||||
{
|
||||
self.price_to_amount.decrement(price, supply);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compute_unrealized_states(
|
||||
&self,
|
||||
height_price: Dollars,
|
||||
date_price: Option<Dollars>,
|
||||
) -> (UnrealizedState, Option<UnrealizedState>) {
|
||||
if self.price_to_amount.is_empty() {
|
||||
return (
|
||||
UnrealizedState::NAN,
|
||||
date_price.map(|_| UnrealizedState::NAN),
|
||||
);
|
||||
}
|
||||
|
||||
let mut height_unrealized_state = UnrealizedState::ZERO;
|
||||
let mut date_unrealized_state = date_price.map(|_| UnrealizedState::ZERO);
|
||||
|
||||
let update_state =
|
||||
|price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
|
||||
match price.cmp(¤t_price) {
|
||||
Ordering::Equal => {
|
||||
state.supply_even += sats;
|
||||
}
|
||||
Ordering::Less => {
|
||||
state.supply_in_profit += sats;
|
||||
if price > Dollars::ZERO && current_price > Dollars::ZERO {
|
||||
let diff = current_price.checked_sub(price).unwrap();
|
||||
if diff <= Dollars::ZERO {
|
||||
dbg!(price, current_price, diff, sats);
|
||||
panic!();
|
||||
}
|
||||
state.unrealized_profit += diff * sats;
|
||||
}
|
||||
}
|
||||
Ordering::Greater => {
|
||||
state.supply_in_loss += sats;
|
||||
if price > Dollars::ZERO && current_price > Dollars::ZERO {
|
||||
let diff = price.checked_sub(current_price).unwrap();
|
||||
if diff <= Dollars::ZERO {
|
||||
dbg!(price, current_price, diff, sats);
|
||||
panic!();
|
||||
}
|
||||
state.unrealized_loss += diff * sats;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
self.price_to_amount.iter().for_each(|(&price, &sats)| {
|
||||
update_state(price, height_price, sats, &mut height_unrealized_state);
|
||||
|
||||
if let Some(date_price) = date_price {
|
||||
update_state(
|
||||
price,
|
||||
date_price,
|
||||
sats,
|
||||
date_unrealized_state.as_mut().unwrap(),
|
||||
)
|
||||
}
|
||||
});
|
||||
|
||||
(height_unrealized_state, date_unrealized_state)
|
||||
}
|
||||
|
||||
pub fn commit(&mut self, height: Height) -> Result<()> {
|
||||
self.price_to_amount.flush(height)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
mod address;
|
||||
mod any;
|
||||
mod common;
|
||||
mod utxo;
|
||||
|
||||
pub use address::*;
|
||||
pub use any::*;
|
||||
pub use common::*;
|
||||
pub use utxo::*;
|
||||
@@ -0,0 +1,27 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Height, Result};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use super::CohortState;
|
||||
|
||||
#[derive(Clone, Deref, DerefMut)]
|
||||
pub struct UTXOCohortState(CohortState);
|
||||
|
||||
impl UTXOCohortState {
|
||||
pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result<Self> {
|
||||
Ok(Self(CohortState::default_and_import(
|
||||
path,
|
||||
name,
|
||||
compute_dollars,
|
||||
)?))
|
||||
}
|
||||
|
||||
pub fn height(&self) -> Option<Height> {
|
||||
self.0.height()
|
||||
}
|
||||
|
||||
pub fn reset_price_to_amount(&mut self) -> Result<()> {
|
||||
self.0.reset_price_to_amount()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
mod block;
|
||||
mod cohorts;
|
||||
mod price_to_amount;
|
||||
mod realized;
|
||||
mod supply;
|
||||
mod transacted;
|
||||
mod unrealized;
|
||||
|
||||
pub use block::*;
|
||||
pub use cohorts::*;
|
||||
pub use price_to_amount::*;
|
||||
pub use realized::*;
|
||||
pub use supply::*;
|
||||
pub use transacted::*;
|
||||
pub use unrealized::*;
|
||||
@@ -0,0 +1,120 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs::{self, File},
|
||||
io::{BufReader, BufWriter},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use bincode::{Decode, Encode, config, decode_from_std_read, encode_into_std_write};
|
||||
use brk_core::{Dollars, Height, Result, Sats};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::states::SupplyState;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PriceToAmount {
|
||||
pathbuf: PathBuf,
|
||||
height: Option<Height>,
|
||||
state: State,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Debug, Deref, DerefMut, Serialize, Deserialize, Encode, Decode)]
|
||||
struct State(BTreeMap<Dollars, Sats>);
|
||||
|
||||
impl PriceToAmount {
|
||||
pub fn forced_import(path: &Path, name: &str) -> Self {
|
||||
Self::import(path, name).unwrap_or_else(|_| Self {
|
||||
pathbuf: Self::path_(path, name),
|
||||
height: None,
|
||||
state: State::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn import(path: &Path, name: &str) -> Result<Self> {
|
||||
let path = Self::path_(path, name);
|
||||
fs::create_dir_all(&path)?;
|
||||
|
||||
let config = config::standard();
|
||||
let file = File::open(Self::path_state_(&path))?;
|
||||
let mut reader = BufReader::new(file);
|
||||
let state = decode_from_std_read(&mut reader, config)?;
|
||||
|
||||
Ok(Self {
|
||||
height: Height::try_from(Self::path_height_(&path).as_path()).ok(),
|
||||
pathbuf: path,
|
||||
state,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = (&Dollars, &Sats)> {
|
||||
self.state.iter()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.state.is_empty()
|
||||
}
|
||||
|
||||
pub fn first_key_value(&self) -> Option<(&Dollars, &Sats)> {
|
||||
self.state.first_key_value()
|
||||
}
|
||||
|
||||
pub fn last_key_value(&self) -> Option<(&Dollars, &Sats)> {
|
||||
self.state.last_key_value()
|
||||
}
|
||||
|
||||
pub fn increment(&mut self, price: Dollars, supply_state: &SupplyState) {
|
||||
*self.state.entry(price).or_default() += supply_state.value;
|
||||
}
|
||||
|
||||
pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) {
|
||||
let amount = self.state.get_mut(&price).unwrap();
|
||||
*amount -= supply_state.value;
|
||||
if *amount == Sats::ZERO {
|
||||
self.state.remove(&price);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
self.state.clear();
|
||||
self.height = None;
|
||||
fs::remove_dir_all(&self.pathbuf)?;
|
||||
fs::create_dir_all(&self.pathbuf)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn flush(&mut self, height: Height) -> Result<()> {
|
||||
self.height = Some(height);
|
||||
height.write(&self.path_height())?;
|
||||
|
||||
let config = config::standard();
|
||||
let file = File::create(self.path_state()).inspect_err(|_| {
|
||||
dbg!(self.path_state());
|
||||
})?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
encode_into_std_write(&self.state, &mut writer, config)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn height(&self) -> Option<Height> {
|
||||
self.height
|
||||
}
|
||||
|
||||
fn path_(path: &Path, name: &str) -> PathBuf {
|
||||
path.join(format!("{name}_price_to_amount"))
|
||||
}
|
||||
|
||||
fn path_state(&self) -> PathBuf {
|
||||
Self::path_state_(&self.pathbuf)
|
||||
}
|
||||
fn path_state_(path: &Path) -> PathBuf {
|
||||
path.join("state")
|
||||
}
|
||||
|
||||
fn path_height(&self) -> PathBuf {
|
||||
Self::path_height_(&self.pathbuf)
|
||||
}
|
||||
fn path_height_(path: &Path) -> PathBuf {
|
||||
path.join("height")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use brk_core::{CheckedSub, Dollars};
|
||||
|
||||
use super::SupplyState;
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct RealizedState {
|
||||
pub cap: Dollars,
|
||||
pub profit: Dollars,
|
||||
pub loss: Dollars,
|
||||
pub value_created: Dollars,
|
||||
pub adj_value_created: Dollars,
|
||||
pub value_destroyed: Dollars,
|
||||
pub adj_value_destroyed: Dollars,
|
||||
}
|
||||
|
||||
impl RealizedState {
|
||||
pub const NAN: Self = Self {
|
||||
cap: Dollars::NAN,
|
||||
profit: Dollars::NAN,
|
||||
loss: Dollars::NAN,
|
||||
value_created: Dollars::NAN,
|
||||
adj_value_created: Dollars::NAN,
|
||||
value_destroyed: Dollars::NAN,
|
||||
adj_value_destroyed: Dollars::NAN,
|
||||
};
|
||||
|
||||
pub fn reset_single_iteration_values(&mut self) {
|
||||
if self.cap != Dollars::NAN {
|
||||
self.profit = Dollars::ZERO;
|
||||
self.loss = Dollars::ZERO;
|
||||
self.value_created = Dollars::ZERO;
|
||||
self.adj_value_created = Dollars::ZERO;
|
||||
self.value_destroyed = Dollars::ZERO;
|
||||
self.adj_value_destroyed = Dollars::ZERO;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn increment(&mut self, supply_state: &SupplyState, price: Dollars) {
|
||||
if supply_state.value.is_zero() {
|
||||
return;
|
||||
}
|
||||
|
||||
self.increment_(price * supply_state.value)
|
||||
}
|
||||
|
||||
pub fn increment_(&mut self, realized_cap: Dollars) {
|
||||
if self.cap == Dollars::NAN {
|
||||
self.cap = Dollars::ZERO;
|
||||
self.profit = Dollars::ZERO;
|
||||
self.loss = Dollars::ZERO;
|
||||
self.value_created = Dollars::ZERO;
|
||||
self.adj_value_created = Dollars::ZERO;
|
||||
self.value_destroyed = Dollars::ZERO;
|
||||
self.adj_value_destroyed = Dollars::ZERO;
|
||||
}
|
||||
|
||||
self.cap += realized_cap;
|
||||
}
|
||||
|
||||
pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) {
|
||||
self.decrement_(price * supply_state.value);
|
||||
}
|
||||
|
||||
pub fn decrement_(&mut self, realized_cap: Dollars) {
|
||||
self.cap = self.cap.checked_sub(realized_cap).unwrap();
|
||||
}
|
||||
|
||||
pub fn receive(&mut self, supply_state: &SupplyState, current_price: Dollars) {
|
||||
self.increment(supply_state, current_price);
|
||||
}
|
||||
|
||||
pub fn send(
|
||||
&mut self,
|
||||
supply_state: &SupplyState,
|
||||
current_price: Dollars,
|
||||
prev_price: Dollars,
|
||||
older_than_hour: bool,
|
||||
) {
|
||||
let current_value = current_price * supply_state.value;
|
||||
let prev_value = prev_price * supply_state.value;
|
||||
|
||||
self.value_created += current_value;
|
||||
self.value_destroyed += prev_value;
|
||||
|
||||
if older_than_hour {
|
||||
self.adj_value_created += current_value;
|
||||
self.adj_value_destroyed += prev_value;
|
||||
}
|
||||
|
||||
match current_price.cmp(&prev_price) {
|
||||
Ordering::Greater => {
|
||||
self.profit += current_value.checked_sub(prev_value).unwrap();
|
||||
}
|
||||
Ordering::Less => {
|
||||
self.loss += prev_value.checked_sub(current_value).unwrap();
|
||||
}
|
||||
Ordering::Equal => {}
|
||||
}
|
||||
|
||||
self.decrement(supply_state, prev_price);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
use std::ops::{Add, AddAssign, SubAssign};
|
||||
|
||||
use brk_core::{AddressData, CheckedSub, Sats};
|
||||
use serde::Serialize;
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
|
||||
pub struct SupplyState {
|
||||
pub utxos: usize,
|
||||
pub value: Sats,
|
||||
}
|
||||
|
||||
impl Add<SupplyState> for SupplyState {
|
||||
type Output = Self;
|
||||
fn add(self, rhs: SupplyState) -> Self::Output {
|
||||
Self {
|
||||
utxos: self.utxos + rhs.utxos,
|
||||
value: self.value + rhs.value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AddAssign<SupplyState> for SupplyState {
|
||||
fn add_assign(&mut self, rhs: Self) {
|
||||
*self += &rhs;
|
||||
}
|
||||
}
|
||||
|
||||
impl AddAssign<&SupplyState> for SupplyState {
|
||||
fn add_assign(&mut self, rhs: &Self) {
|
||||
self.utxos += rhs.utxos;
|
||||
self.value += rhs.value;
|
||||
}
|
||||
}
|
||||
|
||||
impl SubAssign<&SupplyState> for SupplyState {
|
||||
fn sub_assign(&mut self, rhs: &Self) {
|
||||
self.utxos = self.utxos.checked_sub(rhs.utxos).unwrap();
|
||||
self.value = self.value.checked_sub(rhs.value).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&AddressData> for SupplyState {
|
||||
fn from(value: &AddressData) -> Self {
|
||||
Self {
|
||||
utxos: value.outputs_len as usize,
|
||||
value: value.amount(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
use std::ops::{Add, AddAssign};
|
||||
|
||||
use brk_core::{ByAmountRange, GroupedByType, OutputType, Sats};
|
||||
|
||||
use super::SupplyState;
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub struct Transacted {
|
||||
pub spendable_supply: SupplyState,
|
||||
pub by_type: GroupedByType<SupplyState>,
|
||||
pub by_size_group: ByAmountRange<SupplyState>,
|
||||
}
|
||||
|
||||
impl Transacted {
|
||||
#[allow(clippy::inconsistent_digit_grouping)]
|
||||
pub fn iterate(&mut self, value: Sats, _type: OutputType) {
|
||||
let supply = SupplyState { utxos: 1, value };
|
||||
|
||||
*self.by_type.get_mut(_type) += &supply;
|
||||
|
||||
if _type.is_unspendable() {
|
||||
return;
|
||||
}
|
||||
|
||||
self.spendable_supply += &supply;
|
||||
|
||||
*self.by_size_group.get_mut(value) += &supply;
|
||||
}
|
||||
}
|
||||
|
||||
impl Add for Transacted {
|
||||
type Output = Self;
|
||||
fn add(self, rhs: Self) -> Self::Output {
|
||||
Self {
|
||||
spendable_supply: self.spendable_supply + rhs.spendable_supply,
|
||||
by_type: self.by_type + rhs.by_type,
|
||||
by_size_group: self.by_size_group + rhs.by_size_group,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AddAssign for Transacted {
|
||||
fn add_assign(&mut self, rhs: Self) {
|
||||
self.by_size_group += rhs.by_size_group;
|
||||
self.spendable_supply += &rhs.spendable_supply;
|
||||
self.by_type += rhs.by_type;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
use brk_core::{Dollars, Sats};
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct UnrealizedState {
|
||||
pub supply_in_profit: Sats,
|
||||
pub supply_even: Sats,
|
||||
pub supply_in_loss: Sats,
|
||||
pub unrealized_profit: Dollars,
|
||||
pub unrealized_loss: Dollars,
|
||||
}
|
||||
|
||||
impl UnrealizedState {
|
||||
pub const NAN: Self = Self {
|
||||
supply_in_profit: Sats::ZERO,
|
||||
supply_even: Sats::ZERO,
|
||||
supply_in_loss: Sats::ZERO,
|
||||
unrealized_profit: Dollars::NAN,
|
||||
unrealized_loss: Dollars::NAN,
|
||||
};
|
||||
|
||||
pub const ZERO: Self = Self {
|
||||
supply_in_profit: Sats::ZERO,
|
||||
supply_even: Sats::ZERO,
|
||||
supply_in_loss: Sats::ZERO,
|
||||
unrealized_profit: Dollars::ZERO,
|
||||
unrealized_loss: Dollars::ZERO,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,647 @@
|
||||
use std::{path::Path, thread};
|
||||
|
||||
use brk_core::{
|
||||
AddressData, ByAddressType, EmptyAddressData, Height, OutputType, P2AAddressIndex,
|
||||
P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex,
|
||||
P2WPKHAddressIndex, P2WSHAddressIndex, Result, TypeIndex, Version,
|
||||
};
|
||||
use brk_store::{AnyStore, Store};
|
||||
use fjall::{PersistMode, TransactionalKeyspace};
|
||||
use log::info;
|
||||
|
||||
use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Stores {
|
||||
keyspace: TransactionalKeyspace,
|
||||
|
||||
pub p2aaddressindex_to_addressdata: Store<P2AAddressIndex, AddressData>,
|
||||
pub p2aaddressindex_to_emptyaddressdata: Store<P2AAddressIndex, EmptyAddressData>,
|
||||
pub p2pk33addressindex_to_addressdata: Store<P2PK33AddressIndex, AddressData>,
|
||||
pub p2pk33addressindex_to_emptyaddressdata: Store<P2PK33AddressIndex, EmptyAddressData>,
|
||||
pub p2pk65addressindex_to_addressdata: Store<P2PK65AddressIndex, AddressData>,
|
||||
pub p2pk65addressindex_to_emptyaddressdata: Store<P2PK65AddressIndex, EmptyAddressData>,
|
||||
pub p2pkhaddressindex_to_addressdata: Store<P2PKHAddressIndex, AddressData>,
|
||||
pub p2pkhaddressindex_to_emptyaddressdata: Store<P2PKHAddressIndex, EmptyAddressData>,
|
||||
pub p2shaddressindex_to_addressdata: Store<P2SHAddressIndex, AddressData>,
|
||||
pub p2shaddressindex_to_emptyaddressdata: Store<P2SHAddressIndex, EmptyAddressData>,
|
||||
pub p2traddressindex_to_addressdata: Store<P2TRAddressIndex, AddressData>,
|
||||
pub p2traddressindex_to_emptyaddressdata: Store<P2TRAddressIndex, EmptyAddressData>,
|
||||
pub p2wpkhaddressindex_to_addressdata: Store<P2WPKHAddressIndex, AddressData>,
|
||||
pub p2wpkhaddressindex_to_emptyaddressdata: Store<P2WPKHAddressIndex, EmptyAddressData>,
|
||||
pub p2wshaddressindex_to_addressdata: Store<P2WSHAddressIndex, AddressData>,
|
||||
pub p2wshaddressindex_to_emptyaddressdata: Store<P2WSHAddressIndex, EmptyAddressData>,
|
||||
}
|
||||
|
||||
impl Stores {
|
||||
pub fn import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
keyspace: &TransactionalKeyspace,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let (
|
||||
(p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata),
|
||||
(p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata),
|
||||
(p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata),
|
||||
(p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata),
|
||||
(p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata),
|
||||
(p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata),
|
||||
(p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata),
|
||||
(p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata),
|
||||
) = thread::scope(|scope| {
|
||||
let p2a = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2aaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2aaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2pk33 = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk33addressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk33addressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2pk65 = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk65addressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk65addressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2pkh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pkhaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pkhaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2sh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2shaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2shaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2tr = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2traddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2traddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2wpkh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wpkhaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wpkhaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2wsh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wshaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wshaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
(
|
||||
p2a.join().unwrap(),
|
||||
p2pk33.join().unwrap(),
|
||||
p2pk65.join().unwrap(),
|
||||
p2pkh.join().unwrap(),
|
||||
p2sh.join().unwrap(),
|
||||
p2tr.join().unwrap(),
|
||||
p2wpkh.join().unwrap(),
|
||||
p2wsh.join().unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
keyspace: keyspace.clone(),
|
||||
|
||||
p2aaddressindex_to_addressdata,
|
||||
p2aaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2pk33addressindex_to_addressdata,
|
||||
p2pk33addressindex_to_emptyaddressdata,
|
||||
|
||||
p2pk65addressindex_to_addressdata,
|
||||
p2pk65addressindex_to_emptyaddressdata,
|
||||
|
||||
p2pkhaddressindex_to_addressdata,
|
||||
p2pkhaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2shaddressindex_to_addressdata,
|
||||
p2shaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2traddressindex_to_addressdata,
|
||||
p2traddressindex_to_emptyaddressdata,
|
||||
|
||||
p2wpkhaddressindex_to_addressdata,
|
||||
p2wpkhaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2wshaddressindex_to_addressdata,
|
||||
p2wshaddressindex_to_emptyaddressdata,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn starting_height(&self) -> Height {
|
||||
self.as_slice()
|
||||
.into_iter()
|
||||
.map(|store| store.height().map(Height::incremented).unwrap_or_default())
|
||||
.min()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
info!("Resetting stores...");
|
||||
info!("> If it gets stuck here, stop the program and start it again");
|
||||
|
||||
self.as_mut_slice()
|
||||
.into_iter()
|
||||
.try_for_each(|store| store.reset())?;
|
||||
|
||||
self.keyspace
|
||||
.persist(PersistMode::SyncAll)
|
||||
.map_err(|e| e.into())
|
||||
}
|
||||
|
||||
pub fn get_addressdata(
|
||||
&self,
|
||||
address_type: OutputType,
|
||||
type_index: TypeIndex,
|
||||
) -> Result<Option<AddressData>> {
|
||||
Ok(match address_type {
|
||||
OutputType::P2A => self
|
||||
.p2aaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK33 => self
|
||||
.p2pk33addressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK65 => self
|
||||
.p2pk65addressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PKH => self
|
||||
.p2pkhaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2SH => self
|
||||
.p2shaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2TR => self
|
||||
.p2traddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WPKH => self
|
||||
.p2wpkhaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WSH => self
|
||||
.p2wshaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_emptyaddressdata(
|
||||
&self,
|
||||
address_type: OutputType,
|
||||
type_index: TypeIndex,
|
||||
) -> Result<Option<EmptyAddressData>> {
|
||||
Ok(match address_type {
|
||||
OutputType::P2A => self
|
||||
.p2aaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK33 => self
|
||||
.p2pk33addressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK65 => self
|
||||
.p2pk65addressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PKH => self
|
||||
.p2pkhaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2SH => self
|
||||
.p2shaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2TR => self
|
||||
.p2traddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WPKH => self
|
||||
.p2wpkhaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WSH => self
|
||||
.p2wshaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn commit(
|
||||
&mut self,
|
||||
height: Height,
|
||||
addresstype_to_typeindex_to_addressdata: AddressTypeToTypeIndexTree<
|
||||
WithAddressDataSource<AddressData>,
|
||||
>,
|
||||
addresstype_to_typeindex_to_emptyaddressdata: AddressTypeToTypeIndexTree<
|
||||
WithAddressDataSource<EmptyAddressData>,
|
||||
>,
|
||||
) -> Result<()> {
|
||||
let ByAddressType {
|
||||
p2pk65,
|
||||
p2pk33,
|
||||
p2pkh,
|
||||
p2sh,
|
||||
p2wpkh,
|
||||
p2wsh,
|
||||
p2tr,
|
||||
p2a,
|
||||
} = addresstype_to_typeindex_to_addressdata.unwrap();
|
||||
|
||||
let ByAddressType {
|
||||
p2pk65: empty_p2pk65,
|
||||
p2pk33: empty_p2pk33,
|
||||
p2pkh: empty_p2pkh,
|
||||
p2sh: empty_p2sh,
|
||||
p2wpkh: empty_p2wpkh,
|
||||
p2wsh: empty_p2wsh,
|
||||
p2tr: empty_p2tr,
|
||||
p2a: empty_p2a,
|
||||
} = addresstype_to_typeindex_to_emptyaddressdata.unwrap();
|
||||
|
||||
thread::scope(|s| {
|
||||
s.spawn(|| {
|
||||
self.p2aaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2a
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2a.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2pk33addressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2pk33
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2pk33.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2pk65addressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2pk65
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2pk65.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2pkhaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2pkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2pkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2shaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2sh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2sh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2traddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2tr
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2tr.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2wpkhaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2wpkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2wpkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2wshaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2wsh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2wsh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
});
|
||||
|
||||
thread::scope(|scope| {
|
||||
scope.spawn(|| {
|
||||
self.p2aaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2a.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2a.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2pk33addressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2pk33
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2pk33.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2pk65addressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2pk65
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2pk65.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2pkhaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2pkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2pkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2shaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2sh.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2sh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2traddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2tr.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2tr.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2wpkhaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2wpkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2wpkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2wshaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2wsh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2wsh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
});
|
||||
|
||||
self.keyspace
|
||||
.persist(PersistMode::SyncAll)
|
||||
.map_err(|e| e.into())
|
||||
}
|
||||
|
||||
pub fn rotate_memtables(&self) {
|
||||
self.as_slice()
|
||||
.into_iter()
|
||||
.for_each(|store| store.rotate_memtable());
|
||||
}
|
||||
|
||||
pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] {
|
||||
[
|
||||
&self.p2aaddressindex_to_addressdata,
|
||||
&self.p2aaddressindex_to_emptyaddressdata,
|
||||
&self.p2pk33addressindex_to_addressdata,
|
||||
&self.p2pk33addressindex_to_emptyaddressdata,
|
||||
&self.p2pk65addressindex_to_addressdata,
|
||||
&self.p2pk65addressindex_to_emptyaddressdata,
|
||||
&self.p2pkhaddressindex_to_addressdata,
|
||||
&self.p2pkhaddressindex_to_emptyaddressdata,
|
||||
&self.p2shaddressindex_to_addressdata,
|
||||
&self.p2shaddressindex_to_emptyaddressdata,
|
||||
&self.p2traddressindex_to_addressdata,
|
||||
&self.p2traddressindex_to_emptyaddressdata,
|
||||
&self.p2wpkhaddressindex_to_addressdata,
|
||||
&self.p2wpkhaddressindex_to_emptyaddressdata,
|
||||
&self.p2wshaddressindex_to_addressdata,
|
||||
&self.p2wshaddressindex_to_emptyaddressdata,
|
||||
]
|
||||
}
|
||||
|
||||
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 16] {
|
||||
[
|
||||
&mut self.p2aaddressindex_to_addressdata,
|
||||
&mut self.p2aaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2pk33addressindex_to_addressdata,
|
||||
&mut self.p2pk33addressindex_to_emptyaddressdata,
|
||||
&mut self.p2pk65addressindex_to_addressdata,
|
||||
&mut self.p2pk65addressindex_to_emptyaddressdata,
|
||||
&mut self.p2pkhaddressindex_to_addressdata,
|
||||
&mut self.p2pkhaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2shaddressindex_to_addressdata,
|
||||
&mut self.p2shaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2traddressindex_to_addressdata,
|
||||
&mut self.p2traddressindex_to_emptyaddressdata,
|
||||
&mut self.p2wpkhaddressindex_to_addressdata,
|
||||
&mut self.p2wpkhaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2wshaddressindex_to_addressdata,
|
||||
&mut self.p2wshaddressindex_to_emptyaddressdata,
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
use std::ops::{Add, Div};
|
||||
|
||||
pub fn get_percentile<T>(sorted: &[T], percentile: f64) -> T
|
||||
where
|
||||
T: Clone + Div<usize, Output = T> + Add<T, Output = T>,
|
||||
{
|
||||
let len = sorted.len();
|
||||
|
||||
if len == 0 {
|
||||
panic!();
|
||||
} else if len == 1 {
|
||||
sorted[0].clone()
|
||||
} else {
|
||||
let index = (len - 1) as f64 * percentile;
|
||||
|
||||
let fract = index.fract();
|
||||
|
||||
if fract != 0.0 {
|
||||
let left = sorted.get(index as usize).unwrap().clone();
|
||||
let right = sorted.get(index.ceil() as usize).unwrap().clone();
|
||||
left / 2 + right / 2
|
||||
} else {
|
||||
// dbg!(sorted.len(), index);
|
||||
sorted.get(index as usize).unwrap().clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{
|
||||
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
|
||||
Timestamp, Version, Weight,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub height_to_interval: EagerVec<Height, Timestamp>,
|
||||
pub height_to_vbytes: EagerVec<Height, StoredU64>,
|
||||
pub difficultyepoch_to_timestamp: EagerVec<DifficultyEpoch, Timestamp>,
|
||||
pub halvingepoch_to_timestamp: EagerVec<HalvingEpoch, Timestamp>,
|
||||
pub timeindexes_to_timestamp: ComputedVecsFromDateIndex<Timestamp>,
|
||||
pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
|
||||
pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
|
||||
pub indexes_to_block_size: ComputedVecsFromHeight<StoredUsize>,
|
||||
pub indexes_to_block_vbytes: ComputedVecsFromHeight<StoredU64>,
|
||||
pub indexes_to_block_weight: ComputedVecsFromHeight<Weight>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
height_to_interval: EagerVec::forced_import(
|
||||
path,
|
||||
"interval",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"timestamp",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_first(),
|
||||
)?,
|
||||
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_interval",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_percentiles()
|
||||
.add_minmax()
|
||||
.add_average(),
|
||||
)?,
|
||||
indexes_to_block_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_count",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_weight",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_size",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
height_to_vbytes: EagerVec::forced_import(
|
||||
path,
|
||||
"vbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"block_vbytes",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
difficultyepoch_to_timestamp: EagerVec::forced_import(
|
||||
path,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
halvingepoch_to_timestamp: EagerVec::forced_import(
|
||||
path,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.timeindexes_to_timestamp.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, indexes, starting_indexes, exit| {
|
||||
vec.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&indexes.dateindex_to_date,
|
||||
|(di, d, ..)| (di, Timestamp::from(d)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_block_count.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, indexer, _, starting_indexes, exit| {
|
||||
v.compute_range(
|
||||
starting_indexes.height,
|
||||
&indexer.vecs.height_to_weight,
|
||||
|h| (h, StoredU32::from(1_u32)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
let mut height_to_timestamp_iter = indexer.vecs.height_to_timestamp.iter();
|
||||
self.height_to_interval.compute_transform(
|
||||
starting_indexes.height,
|
||||
&indexer.vecs.height_to_timestamp,
|
||||
|(height, timestamp, ..)| {
|
||||
let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| {
|
||||
let prev_timestamp = height_to_timestamp_iter.unwrap_get_inner(prev_h);
|
||||
timestamp
|
||||
.checked_sub(prev_timestamp)
|
||||
.unwrap_or(Timestamp::ZERO)
|
||||
});
|
||||
(height, interval)
|
||||
},
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.indexes_to_block_interval.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_interval),
|
||||
)?;
|
||||
|
||||
self.indexes_to_block_weight.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&indexer.vecs.height_to_weight),
|
||||
)?;
|
||||
|
||||
self.indexes_to_block_size.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&indexer.vecs.height_to_total_size),
|
||||
)?;
|
||||
|
||||
self.height_to_vbytes.compute_transform(
|
||||
starting_indexes.height,
|
||||
&indexer.vecs.height_to_weight,
|
||||
|(h, w, ..)| {
|
||||
(
|
||||
h,
|
||||
StoredU64::from(bitcoin::Weight::from(w).to_vbytes_floor()),
|
||||
)
|
||||
},
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.indexes_to_block_vbytes.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_vbytes),
|
||||
)?;
|
||||
|
||||
let mut height_to_timestamp_iter = indexer.vecs.height_to_timestamp.iter();
|
||||
|
||||
self.difficultyepoch_to_timestamp.compute_transform(
|
||||
starting_indexes.difficultyepoch,
|
||||
&indexes.difficultyepoch_to_first_height,
|
||||
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.halvingepoch_to_timestamp.compute_transform(
|
||||
starting_indexes.halvingepoch,
|
||||
&indexes.halvingepoch_to_first_height,
|
||||
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
vec![
|
||||
&self.height_to_interval as &dyn AnyCollectableVec,
|
||||
&self.height_to_vbytes,
|
||||
&self.difficultyepoch_to_timestamp,
|
||||
&self.halvingepoch_to_timestamp,
|
||||
],
|
||||
self.timeindexes_to_timestamp.vecs(),
|
||||
self.indexes_to_block_count.vecs(),
|
||||
self.indexes_to_block_interval.vecs(),
|
||||
self.indexes_to_block_size.vecs(),
|
||||
self.indexes_to_block_vbytes.vecs(),
|
||||
self.indexes_to_block_weight.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,679 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
|
||||
|
||||
use crate::vecs::{
|
||||
fetched,
|
||||
grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight},
|
||||
stateful, transactions,
|
||||
};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub indexes_to_coinblocks_created: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_coinblocks_stored: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_liveliness: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_vaultedness: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_vaulted_supply: ComputedValueVecsFromHeight,
|
||||
pub indexes_to_active_supply: ComputedValueVecsFromHeight,
|
||||
pub indexes_to_thermo_cap: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_investor_cap: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_vaulted_cap: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_active_cap: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_vaulted_price: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex,
|
||||
pub indexes_to_active_price: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex,
|
||||
pub indexes_to_true_market_mean: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex,
|
||||
pub indexes_to_cointime_value_destroyed: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_cointime_value_created: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_cointime_value_stored: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_cointime_price: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_cointime_cap: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex,
|
||||
// pub indexes_to_thermo_cap_relative_to_investor_cap: ComputedValueVecsFromHeight,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
format: Format,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let compute_dollars = fetched.is_some();
|
||||
|
||||
Ok(Self {
|
||||
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"coinblocks_created",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"coinblocks_stored",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"liveliness",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"vaultedness",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"activity_to_vaultedness_ratio",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
"vaulted_supply",
|
||||
true,
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
"active_supply",
|
||||
true,
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"thermo_cap",
|
||||
true,
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"investor_cap",
|
||||
true,
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"vaulted_cap",
|
||||
true,
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"active_cap",
|
||||
true,
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"vaulted_price",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"vaulted_price",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_active_price: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"active_price",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"active_price",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"true_market_mean",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"true_market_mean",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"cointime_value_destroyed",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"cointime_value_created",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"cointime_value_stored",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default()
|
||||
.add_sum()
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"cointime_price",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"cointime_cap",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"cointime_price",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
transactions: &transactions::Vecs,
|
||||
stateful: &stateful::Vecs,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply;
|
||||
|
||||
self.indexes_to_coinblocks_created.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_transform(
|
||||
starting_indexes.height,
|
||||
circulating_supply,
|
||||
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
let indexes_to_coinblocks_destroyed =
|
||||
&stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed;
|
||||
|
||||
self.indexes_to_coinblocks_stored.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
let mut coinblocks_destroyed_iter = indexes_to_coinblocks_destroyed
|
||||
.height
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.into_iter();
|
||||
vec.compute_transform(
|
||||
starting_indexes.height,
|
||||
self.indexes_to_coinblocks_created.height.as_ref().unwrap(),
|
||||
|(i, created, ..)| {
|
||||
let destroyed = coinblocks_destroyed_iter.unwrap_get_inner(i);
|
||||
(i, created.checked_sub(destroyed).unwrap())
|
||||
},
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_liveliness.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_divide(
|
||||
starting_indexes.height,
|
||||
indexes_to_coinblocks_destroyed
|
||||
.height_extra
|
||||
.unwrap_cumulative(),
|
||||
self.indexes_to_coinblocks_created
|
||||
.height_extra
|
||||
.unwrap_cumulative(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
let liveliness = &self.indexes_to_liveliness;
|
||||
|
||||
self.indexes_to_vaultedness.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_transform(
|
||||
starting_indexes.height,
|
||||
liveliness.height.as_ref().unwrap(),
|
||||
|(i, v, ..)| (i, StoredF64::from(1.0).checked_sub(v).unwrap()),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
let vaultedness = &self.indexes_to_vaultedness;
|
||||
|
||||
self.indexes_to_activity_to_vaultedness_ratio.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_divide(
|
||||
starting_indexes.height,
|
||||
liveliness.height.as_ref().unwrap(),
|
||||
vaultedness.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_vaulted_supply.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
circulating_supply,
|
||||
vaultedness.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_active_supply.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
circulating_supply,
|
||||
liveliness.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
if let Some(fetched) = fetched {
|
||||
let realized_cap = stateful
|
||||
.utxo_vecs
|
||||
.all
|
||||
.1
|
||||
.height_to_realized_cap
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
|
||||
let realized_price = stateful
|
||||
.utxo_vecs
|
||||
.all
|
||||
.1
|
||||
.indexes_to_realized_price
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.height
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
|
||||
self.indexes_to_thermo_cap.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_transform(
|
||||
starting_indexes.height,
|
||||
transactions
|
||||
.indexes_to_subsidy
|
||||
.dollars
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.height_extra
|
||||
.unwrap_cumulative(),
|
||||
|(i, v, ..)| (i, v),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_investor_cap.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_subtract(
|
||||
starting_indexes.height,
|
||||
realized_cap,
|
||||
self.indexes_to_thermo_cap.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_vaulted_cap.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_divide(
|
||||
starting_indexes.height,
|
||||
realized_cap,
|
||||
self.indexes_to_vaultedness.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_active_cap.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
realized_cap,
|
||||
self.indexes_to_liveliness.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_vaulted_price.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_divide(
|
||||
starting_indexes.height,
|
||||
realized_price,
|
||||
self.indexes_to_vaultedness.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_vaulted_price_ratio.compute_rest(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(self.indexes_to_vaulted_price.dateindex.unwrap_last()),
|
||||
)?;
|
||||
|
||||
self.indexes_to_active_price.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
realized_price,
|
||||
self.indexes_to_liveliness.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_active_price_ratio.compute_rest(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(self.indexes_to_active_price.dateindex.unwrap_last()),
|
||||
)?;
|
||||
|
||||
self.indexes_to_true_market_mean.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_divide(
|
||||
starting_indexes.height,
|
||||
self.indexes_to_investor_cap.height.as_ref().unwrap(),
|
||||
self.indexes_to_active_supply
|
||||
.bitcoin
|
||||
.height
|
||||
.as_ref()
|
||||
.unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_true_market_mean_ratio.compute_rest(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(self.indexes_to_true_market_mean.dateindex.unwrap_last()),
|
||||
)?;
|
||||
|
||||
self.indexes_to_cointime_value_destroyed.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
// TODO: Another example when the callback should be applied to each index, instead of to base then merging from more granular to less
|
||||
// The price taken won't be correct for time based indexes
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
&fetched.chainindexes_to_close.height,
|
||||
indexes_to_coinblocks_destroyed.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_cointime_value_created.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
&fetched.chainindexes_to_close.height,
|
||||
self.indexes_to_coinblocks_created.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_cointime_value_stored.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
&fetched.chainindexes_to_close.height,
|
||||
self.indexes_to_coinblocks_stored.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_cointime_price.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_divide(
|
||||
starting_indexes.height,
|
||||
self.indexes_to_cointime_value_destroyed
|
||||
.height_extra
|
||||
.unwrap_cumulative(),
|
||||
self.indexes_to_coinblocks_stored
|
||||
.height_extra
|
||||
.unwrap_cumulative(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_cointime_cap.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, _, starting_indexes, exit| {
|
||||
vec.compute_multiply(
|
||||
starting_indexes.height,
|
||||
self.indexes_to_cointime_price.height.as_ref().unwrap(),
|
||||
circulating_supply,
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_cointime_price_ratio.compute_rest(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(self.indexes_to_cointime_price.dateindex.unwrap_last()),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.indexes_to_coinblocks_created.vecs(),
|
||||
self.indexes_to_coinblocks_stored.vecs(),
|
||||
self.indexes_to_liveliness.vecs(),
|
||||
self.indexes_to_vaultedness.vecs(),
|
||||
self.indexes_to_activity_to_vaultedness_ratio.vecs(),
|
||||
self.indexes_to_vaulted_supply.vecs(),
|
||||
self.indexes_to_active_supply.vecs(),
|
||||
self.indexes_to_thermo_cap.vecs(),
|
||||
self.indexes_to_investor_cap.vecs(),
|
||||
self.indexes_to_vaulted_cap.vecs(),
|
||||
self.indexes_to_active_cap.vecs(),
|
||||
self.indexes_to_vaulted_price.vecs(),
|
||||
self.indexes_to_vaulted_price_ratio.vecs(),
|
||||
self.indexes_to_active_price.vecs(),
|
||||
self.indexes_to_active_price_ratio.vecs(),
|
||||
self.indexes_to_true_market_mean.vecs(),
|
||||
self.indexes_to_true_market_mean_ratio.vecs(),
|
||||
self.indexes_to_cointime_price.vecs(),
|
||||
self.indexes_to_cointime_cap.vecs(),
|
||||
self.indexes_to_cointime_price_ratio.vecs(),
|
||||
self.indexes_to_cointime_value_destroyed.vecs(),
|
||||
self.indexes_to_cointime_value_created.vecs(),
|
||||
self.indexes_to_cointime_value_stored.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{StoredU8, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub constant_0: ComputedVecsFromHeight<StoredU8>,
|
||||
pub constant_1: ComputedVecsFromHeight<StoredU8>,
|
||||
pub constant_50: ComputedVecsFromHeight<StoredU8>,
|
||||
pub constant_100: ComputedVecsFromHeight<StoredU8>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
constant_0: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"constant_0",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
constant_1: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"constant_1",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
constant_50: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"constant_50",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
constant_100: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"constant_100",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.constant_0.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, indexes, starting_indexes, exit| {
|
||||
vec.compute_to(
|
||||
starting_indexes.height,
|
||||
indexes.height_to_date.len(),
|
||||
indexes.height_to_date.version(),
|
||||
|i| (i, StoredU8::new(0)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.constant_1.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, indexes, starting_indexes, exit| {
|
||||
vec.compute_to(
|
||||
starting_indexes.height,
|
||||
indexes.height_to_date.len(),
|
||||
indexes.height_to_date.version(),
|
||||
|i| (i, StoredU8::new(1)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.constant_50.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, indexes, starting_indexes, exit| {
|
||||
vec.compute_to(
|
||||
starting_indexes.height,
|
||||
indexes.height_to_date.len(),
|
||||
indexes.height_to_date.version(),
|
||||
|i| (i, StoredU8::new(50)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.constant_100.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, indexes, starting_indexes, exit| {
|
||||
vec.compute_to(
|
||||
starting_indexes.height,
|
||||
indexes.height_to_date.len(),
|
||||
indexes.height_to_date.version(),
|
||||
|i| (i, StoredU8::new(100)),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.constant_0.vecs(),
|
||||
self.constant_1.vecs(),
|
||||
self.constant_50.vecs(),
|
||||
self.constant_100.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,901 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{CheckedSub, Result, StoredUsize, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType};
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
|
||||
use crate::utils::get_percentile;
|
||||
|
||||
use super::ComputedType;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ComputedVecBuilder<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: ComputedType,
|
||||
{
|
||||
pub first: Option<Box<EagerVec<I, T>>>,
|
||||
pub average: Option<Box<EagerVec<I, T>>>,
|
||||
pub sum: Option<Box<EagerVec<I, T>>>,
|
||||
pub max: Option<Box<EagerVec<I, T>>>,
|
||||
pub _90p: Option<Box<EagerVec<I, T>>>,
|
||||
pub _75p: Option<Box<EagerVec<I, T>>>,
|
||||
pub median: Option<Box<EagerVec<I, T>>>,
|
||||
pub _25p: Option<Box<EagerVec<I, T>>>,
|
||||
pub _10p: Option<Box<EagerVec<I, T>>>,
|
||||
pub min: Option<Box<EagerVec<I, T>>>,
|
||||
pub last: Option<Box<EagerVec<I, T>>>,
|
||||
pub cumulative: Option<Box<EagerVec<I, T>>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl<I, T> ComputedVecBuilder<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: ComputedType,
|
||||
{
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let only_one_active = options.is_only_one_active();
|
||||
|
||||
let suffix = |s: &str| format!("{name}_{s}");
|
||||
|
||||
let maybe_suffix = |s: &str| {
|
||||
if only_one_active {
|
||||
name.to_string()
|
||||
} else {
|
||||
suffix(s)
|
||||
}
|
||||
};
|
||||
|
||||
let s = Self {
|
||||
first: options.first.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("first"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
last: options.last.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(),
|
||||
)
|
||||
}),
|
||||
min: options.min.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("min"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
max: options.max.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("max"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
median: options.median.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("median"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
average: options.average.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("average"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
sum: options.sum.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&(if !options.last && !options.average && !options.min && !options.max {
|
||||
name.to_string()
|
||||
} else {
|
||||
maybe_suffix("sum")
|
||||
}),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
cumulative: options.cumulative.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&suffix("cumulative"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
_90p: options._90p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("90p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
_75p: options._75p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("75p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
_25p: options._25p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("25p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
_10p: options._10p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&maybe_suffix("10p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
};
|
||||
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
pub fn extend(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
source: &impl AnyIterableVec<I, T>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
if self.cumulative.is_none() {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.validate_computed_version_or_reset_file(source.version())?;
|
||||
|
||||
let index = self.starting_index(max_from);
|
||||
|
||||
let cumulative_vec = self.cumulative.as_mut().unwrap();
|
||||
|
||||
let mut cumulative = index.decremented().map_or(T::from(0_usize), |index| {
|
||||
cumulative_vec.iter().unwrap_get_inner(index)
|
||||
});
|
||||
source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
|
||||
cumulative = cumulative.clone() + v.into_owned();
|
||||
cumulative_vec.forced_push_at(i, cumulative.clone(), exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute<I2>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
source: &impl AnyIterableVec<I2, T>,
|
||||
first_indexes: &impl AnyIterableVec<I, I2>,
|
||||
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I2: StoredIndex + StoredType + CheckedSub<I2>,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
source.version() + first_indexes.version() + count_indexes.version(),
|
||||
)?;
|
||||
|
||||
let index = self.starting_index(max_from);
|
||||
|
||||
let mut count_indexes_iter = count_indexes.iter();
|
||||
let mut source_iter = source.iter();
|
||||
|
||||
let cumulative_vec = self.cumulative.as_mut();
|
||||
|
||||
let mut cumulative = cumulative_vec.map(|cumulative_vec| {
|
||||
index.decremented().map_or(T::from(0_usize), |index| {
|
||||
cumulative_vec.iter().unwrap_get_inner(index)
|
||||
})
|
||||
});
|
||||
|
||||
first_indexes
|
||||
.iter_at(index)
|
||||
.try_for_each(|(i, first_index)| -> Result<()> {
|
||||
let first_index = first_index.into_owned();
|
||||
|
||||
let count_index = count_indexes_iter.unwrap_get_inner(i);
|
||||
|
||||
if let Some(first) = self.first.as_mut() {
|
||||
let f = source_iter
|
||||
.get_inner(first_index)
|
||||
.unwrap_or_else(|| T::from(0_usize));
|
||||
first.forced_push_at(index, f, exit)?;
|
||||
}
|
||||
|
||||
if let Some(last) = self.last.as_mut() {
|
||||
let count_index = *count_index;
|
||||
if count_index == 0 {
|
||||
panic!("should compute last if count can be 0")
|
||||
}
|
||||
let last_index = first_index + (count_index - 1);
|
||||
let v = source_iter.unwrap_get_inner(last_index);
|
||||
// .context("to work")
|
||||
// .inspect_err(|_| {
|
||||
// dbg!(first_index, count_index, last_index);
|
||||
// })
|
||||
// .unwrap()
|
||||
// .into_owned();
|
||||
last.forced_push_at(index, v, exit)?;
|
||||
}
|
||||
|
||||
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
|
||||
let needs_average_sum_or_cumulative =
|
||||
needs_sum_or_cumulative || self.average.is_some();
|
||||
let needs_sorted = self.max.is_some()
|
||||
|| self._90p.is_some()
|
||||
|| self._75p.is_some()
|
||||
|| self.median.is_some()
|
||||
|| self._25p.is_some()
|
||||
|| self._10p.is_some()
|
||||
|| self.min.is_some();
|
||||
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
|
||||
|
||||
if needs_values {
|
||||
source_iter.set(first_index);
|
||||
let mut values = (&mut source_iter)
|
||||
.take(*count_index)
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if needs_sorted {
|
||||
values.sort_unstable();
|
||||
|
||||
if let Some(max) = self.max.as_mut() {
|
||||
max.forced_push_at(
|
||||
i,
|
||||
values
|
||||
.last()
|
||||
.context("expect some")
|
||||
.inspect_err(|_| {
|
||||
dbg!(
|
||||
&values,
|
||||
max.path(),
|
||||
first_indexes.name(),
|
||||
first_index,
|
||||
count_indexes.name(),
|
||||
count_index,
|
||||
source.len(),
|
||||
source.name()
|
||||
);
|
||||
})
|
||||
.unwrap()
|
||||
.clone(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(_90p) = self._90p.as_mut() {
|
||||
_90p.forced_push_at(i, get_percentile(&values, 0.90), exit)?;
|
||||
}
|
||||
|
||||
if let Some(_75p) = self._75p.as_mut() {
|
||||
_75p.forced_push_at(i, get_percentile(&values, 0.75), exit)?;
|
||||
}
|
||||
|
||||
if let Some(median) = self.median.as_mut() {
|
||||
median.forced_push_at(i, get_percentile(&values, 0.50), exit)?;
|
||||
}
|
||||
|
||||
if let Some(_25p) = self._25p.as_mut() {
|
||||
_25p.forced_push_at(i, get_percentile(&values, 0.25), exit)?;
|
||||
}
|
||||
|
||||
if let Some(_10p) = self._10p.as_mut() {
|
||||
_10p.forced_push_at(i, get_percentile(&values, 0.10), exit)?;
|
||||
}
|
||||
|
||||
if let Some(min) = self.min.as_mut() {
|
||||
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
|
||||
}
|
||||
}
|
||||
|
||||
if needs_average_sum_or_cumulative {
|
||||
let len = values.len();
|
||||
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
|
||||
|
||||
if let Some(average) = self.average.as_mut() {
|
||||
let avg = sum.clone() / len;
|
||||
average.forced_push_at(i, avg, exit)?;
|
||||
}
|
||||
|
||||
if needs_sum_or_cumulative {
|
||||
if let Some(sum_vec) = self.sum.as_mut() {
|
||||
sum_vec.forced_push_at(i, sum.clone(), exit)?;
|
||||
}
|
||||
|
||||
if let Some(cumulative_vec) = self.cumulative.as_mut() {
|
||||
let t = cumulative.as_ref().unwrap().clone() + sum;
|
||||
cumulative.replace(t.clone());
|
||||
cumulative_vec.forced_push_at(i, t, exit)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::wrong_self_convention)]
|
||||
pub fn from_aligned<I2>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
source: &ComputedVecBuilder<I2, T>,
|
||||
first_indexes: &impl AnyIterableVec<I, I2>,
|
||||
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I2: StoredIndex + StoredType + CheckedSub<I2>,
|
||||
{
|
||||
if self._90p.is_some()
|
||||
|| self._75p.is_some()
|
||||
|| self.median.is_some()
|
||||
|| self._25p.is_some()
|
||||
|| self._10p.is_some()
|
||||
{
|
||||
panic!("unsupported");
|
||||
}
|
||||
|
||||
self.validate_computed_version_or_reset_file(
|
||||
VERSION + first_indexes.version() + count_indexes.version(),
|
||||
)?;
|
||||
|
||||
let index = self.starting_index(max_from);
|
||||
|
||||
let mut count_indexes_iter = count_indexes.iter();
|
||||
|
||||
let mut source_first_iter = source.first.as_ref().map(|f| f.iter());
|
||||
let mut source_last_iter = source.last.as_ref().map(|f| f.iter());
|
||||
let mut source_max_iter = source.max.as_ref().map(|f| f.iter());
|
||||
let mut source_min_iter = source.min.as_ref().map(|f| f.iter());
|
||||
let mut source_average_iter = source.average.as_ref().map(|f| f.iter());
|
||||
let mut source_sum_iter = source.sum.as_ref().map(|f| f.iter());
|
||||
|
||||
let mut cumulative = self.cumulative.as_mut().map(|cumulative_vec| {
|
||||
index.decremented().map_or(T::from(0_usize), |index| {
|
||||
cumulative_vec.iter().unwrap_get_inner(index)
|
||||
})
|
||||
});
|
||||
|
||||
first_indexes
|
||||
.iter_at(index)
|
||||
.try_for_each(|(i, first_index, ..)| -> Result<()> {
|
||||
let first_index = first_index.into_owned();
|
||||
|
||||
let count_index = count_indexes_iter.unwrap_get_inner(i);
|
||||
|
||||
if let Some(first) = self.first.as_mut() {
|
||||
let v = source_first_iter
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.unwrap_get_inner(first_index);
|
||||
first.forced_push_at(index, v, exit)?;
|
||||
}
|
||||
|
||||
if let Some(last) = self.last.as_mut() {
|
||||
let count_index = *count_index;
|
||||
if count_index == 0 {
|
||||
panic!("should compute last if count can be 0")
|
||||
}
|
||||
let last_index = first_index + (count_index - 1);
|
||||
let v = source_last_iter
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.unwrap_get_inner(last_index);
|
||||
last.forced_push_at(index, v, exit)?;
|
||||
}
|
||||
|
||||
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
|
||||
let needs_average_sum_or_cumulative =
|
||||
needs_sum_or_cumulative || self.average.is_some();
|
||||
let needs_sorted = self.max.is_some() || self.min.is_some();
|
||||
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
|
||||
|
||||
if needs_values {
|
||||
if needs_sorted {
|
||||
if let Some(max) = self.max.as_mut() {
|
||||
let source_max_iter = source_max_iter.as_mut().unwrap();
|
||||
source_max_iter.set(first_index);
|
||||
let mut values = source_max_iter
|
||||
.take(*count_index)
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
values.sort_unstable();
|
||||
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
|
||||
}
|
||||
|
||||
if let Some(min) = self.min.as_mut() {
|
||||
let source_min_iter = source_min_iter.as_mut().unwrap();
|
||||
source_min_iter.set(first_index);
|
||||
let mut values = source_min_iter
|
||||
.take(*count_index)
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
values.sort_unstable();
|
||||
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
|
||||
}
|
||||
}
|
||||
|
||||
if needs_average_sum_or_cumulative {
|
||||
if let Some(average) = self.average.as_mut() {
|
||||
let source_average_iter = source_average_iter.as_mut().unwrap();
|
||||
source_average_iter.set(first_index);
|
||||
let values = source_average_iter
|
||||
.take(*count_index)
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let len = values.len();
|
||||
let cumulative = values.into_iter().fold(T::from(0), |a, b| a + b);
|
||||
// TODO: Multiply by count then divide by cumulative
|
||||
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
|
||||
let avg = cumulative / len;
|
||||
average.forced_push_at(i, avg, exit)?;
|
||||
}
|
||||
|
||||
if needs_sum_or_cumulative {
|
||||
let source_sum_iter = source_sum_iter.as_mut().unwrap();
|
||||
source_sum_iter.set(first_index);
|
||||
let values = source_sum_iter
|
||||
.take(*count_index)
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
|
||||
|
||||
if let Some(sum_vec) = self.sum.as_mut() {
|
||||
sum_vec.forced_push_at(i, sum.clone(), exit)?;
|
||||
}
|
||||
|
||||
if let Some(cumulative_vec) = self.cumulative.as_mut() {
|
||||
let t = cumulative.as_ref().unwrap().clone() + sum;
|
||||
cumulative.replace(t.clone());
|
||||
cumulative_vec.forced_push_at(i, t, exit)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn starting_index(&self, max_from: I) -> I {
|
||||
max_from.min(I::from(
|
||||
self.vecs().into_iter().map(|v| v.len()).min().unwrap(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn unwrap_first(&self) -> &EagerVec<I, T> {
|
||||
self.first.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_average(&self) -> &EagerVec<I, T> {
|
||||
self.average.as_ref().unwrap()
|
||||
}
|
||||
pub fn unwrap_sum(&self) -> &EagerVec<I, T> {
|
||||
self.sum.as_ref().unwrap()
|
||||
}
|
||||
pub fn unwrap_max(&self) -> &EagerVec<I, T> {
|
||||
self.max.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_90p(&self) -> &EagerVec<I, T> {
|
||||
self._90p.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_75p(&self) -> &EagerVec<I, T> {
|
||||
self._75p.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_median(&self) -> &EagerVec<I, T> {
|
||||
self.median.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_25p(&self) -> &EagerVec<I, T> {
|
||||
self._25p.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_10p(&self) -> &EagerVec<I, T> {
|
||||
self._10p.as_ref().unwrap()
|
||||
}
|
||||
pub fn unwrap_min(&self) -> &EagerVec<I, T> {
|
||||
self.min.as_ref().unwrap()
|
||||
}
|
||||
pub fn unwrap_last(&self) -> &EagerVec<I, T> {
|
||||
self.last.as_ref().unwrap()
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn unwrap_cumulative(&self) -> &EagerVec<I, T> {
|
||||
self.cumulative.as_ref().unwrap()
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
let mut v: Vec<&dyn AnyCollectableVec> = vec![];
|
||||
|
||||
if let Some(first) = self.first.as_ref() {
|
||||
v.push(first.as_ref());
|
||||
}
|
||||
if let Some(last) = self.last.as_ref() {
|
||||
v.push(last.as_ref());
|
||||
}
|
||||
if let Some(min) = self.min.as_ref() {
|
||||
v.push(min.as_ref());
|
||||
}
|
||||
if let Some(max) = self.max.as_ref() {
|
||||
v.push(max.as_ref());
|
||||
}
|
||||
if let Some(median) = self.median.as_ref() {
|
||||
v.push(median.as_ref());
|
||||
}
|
||||
if let Some(average) = self.average.as_ref() {
|
||||
v.push(average.as_ref());
|
||||
}
|
||||
if let Some(sum) = self.sum.as_ref() {
|
||||
v.push(sum.as_ref());
|
||||
}
|
||||
if let Some(cumulative) = self.cumulative.as_ref() {
|
||||
v.push(cumulative.as_ref());
|
||||
}
|
||||
if let Some(_90p) = self._90p.as_ref() {
|
||||
v.push(_90p.as_ref());
|
||||
}
|
||||
if let Some(_75p) = self._75p.as_ref() {
|
||||
v.push(_75p.as_ref());
|
||||
}
|
||||
if let Some(_25p) = self._25p.as_ref() {
|
||||
v.push(_25p.as_ref());
|
||||
}
|
||||
if let Some(_10p) = self._10p.as_ref() {
|
||||
v.push(_10p.as_ref());
|
||||
}
|
||||
|
||||
v
|
||||
}
|
||||
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
if let Some(first) = self.first.as_mut() {
|
||||
first.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(last) = self.last.as_mut() {
|
||||
last.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(min) = self.min.as_mut() {
|
||||
min.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(max) = self.max.as_mut() {
|
||||
max.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(median) = self.median.as_mut() {
|
||||
median.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(average) = self.average.as_mut() {
|
||||
average.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(sum) = self.sum.as_mut() {
|
||||
sum.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(cumulative) = self.cumulative.as_mut() {
|
||||
cumulative.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(_90p) = self._90p.as_mut() {
|
||||
_90p.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(_75p) = self._75p.as_mut() {
|
||||
_75p.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(_25p) = self._25p.as_mut() {
|
||||
_25p.safe_flush(exit)?;
|
||||
}
|
||||
if let Some(_10p) = self._10p.as_mut() {
|
||||
_10p.safe_flush(exit)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
|
||||
if let Some(first) = self.first.as_mut() {
|
||||
first.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(last) = self.last.as_mut() {
|
||||
last.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(min) = self.min.as_mut() {
|
||||
min.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(max) = self.max.as_mut() {
|
||||
max.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(median) = self.median.as_mut() {
|
||||
median.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(average) = self.average.as_mut() {
|
||||
average.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(sum) = self.sum.as_mut() {
|
||||
sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(cumulative) = self.cumulative.as_mut() {
|
||||
cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(_90p) = self._90p.as_mut() {
|
||||
_90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(_75p) = self._75p.as_mut() {
|
||||
_75p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(_25p) = self._25p.as_mut() {
|
||||
_25p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
if let Some(_10p) = self._10p.as_mut() {
|
||||
_10p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct StorableVecGeneatorOptions {
|
||||
average: bool,
|
||||
sum: bool,
|
||||
max: bool,
|
||||
_90p: bool,
|
||||
_75p: bool,
|
||||
median: bool,
|
||||
_25p: bool,
|
||||
_10p: bool,
|
||||
min: bool,
|
||||
first: bool,
|
||||
last: bool,
|
||||
cumulative: bool,
|
||||
}
|
||||
|
||||
impl StorableVecGeneatorOptions {
|
||||
pub fn add_first(mut self) -> Self {
|
||||
self.first = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_last(mut self) -> Self {
|
||||
self.last = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_min(mut self) -> Self {
|
||||
self.min = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_max(mut self) -> Self {
|
||||
self.max = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn add_median(mut self) -> Self {
|
||||
self.median = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_average(mut self) -> Self {
|
||||
self.average = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_sum(mut self) -> Self {
|
||||
self.sum = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn add_90p(mut self) -> Self {
|
||||
self._90p = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn add_75p(mut self) -> Self {
|
||||
self._75p = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn add_25p(mut self) -> Self {
|
||||
self._25p = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn add_10p(mut self) -> Self {
|
||||
self._10p = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_cumulative(mut self) -> Self {
|
||||
self.cumulative = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_min(mut self) -> Self {
|
||||
self.min = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_max(mut self) -> Self {
|
||||
self.max = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_median(mut self) -> Self {
|
||||
self.median = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_average(mut self) -> Self {
|
||||
self.average = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_sum(mut self) -> Self {
|
||||
self.sum = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_90p(mut self) -> Self {
|
||||
self._90p = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_75p(mut self) -> Self {
|
||||
self._75p = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_25p(mut self) -> Self {
|
||||
self._25p = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_10p(mut self) -> Self {
|
||||
self._10p = false;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rm_cumulative(mut self) -> Self {
|
||||
self.cumulative = false;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_minmax(mut self) -> Self {
|
||||
self.min = true;
|
||||
self.max = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_percentiles(mut self) -> Self {
|
||||
self._90p = true;
|
||||
self._75p = true;
|
||||
self.median = true;
|
||||
self._25p = true;
|
||||
self._10p = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn remove_percentiles(mut self) -> Self {
|
||||
self._90p = false;
|
||||
self._75p = false;
|
||||
self.median = false;
|
||||
self._25p = false;
|
||||
self._10p = false;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn is_only_one_active(&self) -> bool {
|
||||
[
|
||||
self.average,
|
||||
self.sum,
|
||||
self.max,
|
||||
self._90p,
|
||||
self._75p,
|
||||
self.median,
|
||||
self._25p,
|
||||
self._10p,
|
||||
self.min,
|
||||
self.first,
|
||||
self.last,
|
||||
self.cumulative,
|
||||
]
|
||||
.iter()
|
||||
.filter(|b| **b)
|
||||
.count()
|
||||
== 1
|
||||
}
|
||||
|
||||
pub fn copy_self_extra(&self) -> Self {
|
||||
Self {
|
||||
cumulative: self.cumulative,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{
|
||||
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, Version, WeekIndex, YearIndex,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
|
||||
|
||||
use crate::vecs::{Indexes, indexes};
|
||||
|
||||
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedVecsFromDateIndex<T>
|
||||
where
|
||||
T: ComputedType + PartialOrd,
|
||||
{
|
||||
pub dateindex: Option<EagerVec<DateIndex, T>>,
|
||||
pub dateindex_extra: ComputedVecBuilder<DateIndex, T>,
|
||||
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
|
||||
pub monthindex: ComputedVecBuilder<MonthIndex, T>,
|
||||
pub quarterindex: ComputedVecBuilder<QuarterIndex, T>,
|
||||
pub yearindex: ComputedVecBuilder<YearIndex, T>,
|
||||
pub decadeindex: ComputedVecBuilder<DecadeIndex, T>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl<T> ComputedVecsFromDateIndex<T>
|
||||
where
|
||||
T: ComputedType,
|
||||
{
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let dateindex = compute_source.then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
});
|
||||
|
||||
let dateindex_extra = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options.copy_self_extra(),
|
||||
)?;
|
||||
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
Ok(Self {
|
||||
dateindex,
|
||||
dateindex_extra,
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(
|
||||
&mut EagerVec<DateIndex, T>,
|
||||
&Indexer,
|
||||
&indexes::Vecs,
|
||||
&Indexes,
|
||||
&Exit,
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.dateindex.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let dateindex: Option<&EagerVec<DateIndex, T>> = None;
|
||||
self.compute_rest(indexes, starting_indexes, exit, dateindex)
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(dateindex) = dateindex {
|
||||
self.dateindex_extra
|
||||
.extend(starting_indexes.dateindex, dateindex, exit)?;
|
||||
|
||||
self.weekindex.compute(
|
||||
starting_indexes.weekindex,
|
||||
dateindex,
|
||||
&indexes.weekindex_to_first_dateindex,
|
||||
&indexes.weekindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.monthindex.compute(
|
||||
starting_indexes.monthindex,
|
||||
dateindex,
|
||||
&indexes.monthindex_to_first_dateindex,
|
||||
&indexes.monthindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
} else {
|
||||
let dateindex = self.dateindex.as_ref().unwrap();
|
||||
|
||||
self.dateindex_extra
|
||||
.extend(starting_indexes.dateindex, dateindex, exit)?;
|
||||
|
||||
self.weekindex.compute(
|
||||
starting_indexes.weekindex,
|
||||
dateindex,
|
||||
&indexes.weekindex_to_first_dateindex,
|
||||
&indexes.weekindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.monthindex.compute(
|
||||
starting_indexes.monthindex,
|
||||
dateindex,
|
||||
&indexes.monthindex_to_first_dateindex,
|
||||
&indexes.monthindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
self.quarterindex.from_aligned(
|
||||
starting_indexes.quarterindex,
|
||||
&self.monthindex,
|
||||
&indexes.quarterindex_to_first_monthindex,
|
||||
&indexes.quarterindex_to_monthindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.yearindex.from_aligned(
|
||||
starting_indexes.yearindex,
|
||||
&self.monthindex,
|
||||
&indexes.yearindex_to_first_monthindex,
|
||||
&indexes.yearindex_to_monthindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.decadeindex.from_aligned(
|
||||
starting_indexes.decadeindex,
|
||||
&self.yearindex,
|
||||
&indexes.decadeindex_to_first_yearindex,
|
||||
&indexes.decadeindex_to_yearindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.dateindex
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
|
||||
self.dateindex_extra.vecs(),
|
||||
self.weekindex.vecs(),
|
||||
self.monthindex.vecs(),
|
||||
self.quarterindex.vecs(),
|
||||
self.yearindex.vecs(),
|
||||
self.decadeindex.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{
|
||||
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, Version,
|
||||
WeekIndex, YearIndex,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
|
||||
|
||||
use crate::vecs::{Indexes, indexes};
|
||||
|
||||
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedVecsFromHeight<T>
|
||||
where
|
||||
T: ComputedType + PartialOrd,
|
||||
{
|
||||
pub height: Option<EagerVec<Height, T>>,
|
||||
pub height_extra: ComputedVecBuilder<Height, T>,
|
||||
pub dateindex: ComputedVecBuilder<DateIndex, T>,
|
||||
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
|
||||
pub difficultyepoch: ComputedVecBuilder<DifficultyEpoch, T>,
|
||||
pub monthindex: ComputedVecBuilder<MonthIndex, T>,
|
||||
pub quarterindex: ComputedVecBuilder<QuarterIndex, T>,
|
||||
// 6 months
|
||||
pub yearindex: ComputedVecBuilder<YearIndex, T>,
|
||||
// TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
|
||||
pub decadeindex: ComputedVecBuilder<DecadeIndex, T>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl<T> ComputedVecsFromHeight<T>
|
||||
where
|
||||
T: ComputedType + Ord + From<f64>,
|
||||
f64: From<T>,
|
||||
{
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let height = compute_source.then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
});
|
||||
|
||||
let height_extra = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options.copy_self_extra(),
|
||||
)?;
|
||||
|
||||
let dateindex = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
Ok(Self {
|
||||
height,
|
||||
height_extra,
|
||||
dateindex,
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
difficultyepoch: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(&mut EagerVec<Height, T>, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.height.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let height: Option<&EagerVec<Height, T>> = None;
|
||||
self.compute_rest(indexes, starting_indexes, exit, height)
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
height_vec: Option<&impl AnyIterableVec<Height, T>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(height) = height_vec {
|
||||
self.height_extra
|
||||
.extend(starting_indexes.height, height, exit)?;
|
||||
|
||||
self.dateindex.compute(
|
||||
starting_indexes.dateindex,
|
||||
height,
|
||||
&indexes.dateindex_to_first_height,
|
||||
&indexes.dateindex_to_height_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.difficultyepoch.compute(
|
||||
starting_indexes.difficultyepoch,
|
||||
height,
|
||||
&indexes.difficultyepoch_to_first_height,
|
||||
&indexes.difficultyepoch_to_height_count,
|
||||
exit,
|
||||
)?;
|
||||
} else {
|
||||
let height = self.height.as_ref().unwrap();
|
||||
|
||||
self.height_extra
|
||||
.extend(starting_indexes.height, height, exit)?;
|
||||
|
||||
self.dateindex.compute(
|
||||
starting_indexes.dateindex,
|
||||
height,
|
||||
&indexes.dateindex_to_first_height,
|
||||
&indexes.dateindex_to_height_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.difficultyepoch.compute(
|
||||
starting_indexes.difficultyepoch,
|
||||
height,
|
||||
&indexes.difficultyepoch_to_first_height,
|
||||
&indexes.difficultyepoch_to_height_count,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
self.weekindex.from_aligned(
|
||||
starting_indexes.weekindex,
|
||||
&self.dateindex,
|
||||
&indexes.weekindex_to_first_dateindex,
|
||||
&indexes.weekindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.monthindex.from_aligned(
|
||||
starting_indexes.monthindex,
|
||||
&self.dateindex,
|
||||
&indexes.monthindex_to_first_dateindex,
|
||||
&indexes.monthindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.quarterindex.from_aligned(
|
||||
starting_indexes.quarterindex,
|
||||
&self.monthindex,
|
||||
&indexes.quarterindex_to_first_monthindex,
|
||||
&indexes.quarterindex_to_monthindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.yearindex.from_aligned(
|
||||
starting_indexes.yearindex,
|
||||
&self.monthindex,
|
||||
&indexes.yearindex_to_first_monthindex,
|
||||
&indexes.yearindex_to_monthindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.decadeindex.from_aligned(
|
||||
starting_indexes.decadeindex,
|
||||
&self.yearindex,
|
||||
&indexes.decadeindex_to_first_yearindex,
|
||||
&indexes.decadeindex_to_yearindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.height
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
|
||||
self.height_extra.vecs(),
|
||||
self.dateindex.vecs(),
|
||||
self.weekindex.vecs(),
|
||||
self.difficultyepoch.vecs(),
|
||||
self.monthindex.vecs(),
|
||||
self.quarterindex.vecs(),
|
||||
self.yearindex.vecs(),
|
||||
// self.halvingepoch.vecs(),
|
||||
self.decadeindex.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{DifficultyEpoch, Height, Result, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, EagerVec, Format};
|
||||
|
||||
use crate::vecs::{indexes, Indexes};
|
||||
|
||||
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedVecsFromHeightStrict<T>
|
||||
where
|
||||
T: ComputedType + PartialOrd,
|
||||
{
|
||||
pub height: EagerVec<Height, T>,
|
||||
pub height_extra: ComputedVecBuilder<Height, T>,
|
||||
pub difficultyepoch: ComputedVecBuilder<DifficultyEpoch, T>,
|
||||
// TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl<T> ComputedVecsFromHeightStrict<T>
|
||||
where
|
||||
T: ComputedType + Ord + From<f64>,
|
||||
f64: From<T>,
|
||||
{
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let height =
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?;
|
||||
|
||||
let height_extra = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options.copy_self_extra(),
|
||||
)?;
|
||||
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
Ok(Self {
|
||||
height,
|
||||
height_extra,
|
||||
difficultyepoch: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(&mut EagerVec<Height, T>, &Indexer, &indexes::Vecs, &Indexes, &Exit) -> Result<()>,
|
||||
{
|
||||
compute(&mut self.height, indexer, indexes, starting_indexes, exit)?;
|
||||
|
||||
self.height_extra
|
||||
.extend(starting_indexes.height, &self.height, exit)?;
|
||||
|
||||
self.difficultyepoch.compute(
|
||||
starting_indexes.difficultyepoch,
|
||||
&self.height,
|
||||
&indexes.difficultyepoch_to_first_height,
|
||||
&indexes.difficultyepoch_to_height_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
vec![&self.height as &dyn AnyCollectableVec],
|
||||
self.height_extra.vecs(),
|
||||
self.difficultyepoch.vecs(),
|
||||
// self.halvingepoch.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,629 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{
|
||||
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
|
||||
Result, Sats, TxIndex, Version, WeekIndex, YearIndex,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyVec, CollectableVec, EagerVec, Format, StoredIndex, VecIterator,
|
||||
};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
|
||||
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedVecsFromTxindex<T>
|
||||
where
|
||||
T: ComputedType + PartialOrd,
|
||||
{
|
||||
pub txindex: Option<Box<EagerVec<TxIndex, T>>>,
|
||||
pub height: ComputedVecBuilder<Height, T>,
|
||||
pub dateindex: ComputedVecBuilder<DateIndex, T>,
|
||||
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
|
||||
pub difficultyepoch: ComputedVecBuilder<DifficultyEpoch, T>,
|
||||
pub monthindex: ComputedVecBuilder<MonthIndex, T>,
|
||||
pub quarterindex: ComputedVecBuilder<QuarterIndex, T>,
|
||||
pub yearindex: ComputedVecBuilder<YearIndex, T>,
|
||||
// TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
|
||||
pub decadeindex: ComputedVecBuilder<DecadeIndex, T>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl<T> ComputedVecsFromTxindex<T>
|
||||
where
|
||||
T: ComputedType + Ord + From<f64>,
|
||||
f64: From<T>,
|
||||
{
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let txindex = compute_source.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let height = ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
Ok(Self {
|
||||
txindex,
|
||||
height,
|
||||
dateindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
difficultyepoch: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
// #[allow(unused)]
|
||||
// pub fn compute_all<F>(
|
||||
// &mut self,
|
||||
// indexer: &Indexer,
|
||||
// indexes: &indexes::Vecs,
|
||||
// starting_indexes: &Indexes,
|
||||
// exit: &Exit,
|
||||
// mut compute: F,
|
||||
// ) -> color_eyre::Result<()>
|
||||
// where
|
||||
// F: FnMut(
|
||||
// &mut EagerVec<TxIndex, T>,
|
||||
// &Indexer,
|
||||
// &indexes::Vecs,
|
||||
// &Indexes,
|
||||
// &Exit,
|
||||
// ) -> Result<()>,
|
||||
// {
|
||||
// compute(
|
||||
// self.txindex.as_mut().unwrap(),
|
||||
// indexer,
|
||||
// indexes,
|
||||
// starting_indexes,
|
||||
// exit,
|
||||
// )?;
|
||||
|
||||
// let txindex: Option<&StoredVec<TxIndex, T>> = None;
|
||||
// self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
|
||||
|
||||
// Ok(())
|
||||
// }
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, T>>,
|
||||
) -> Result<()> {
|
||||
if let Some(txindex) = txindex {
|
||||
self.height.compute(
|
||||
starting_indexes.height,
|
||||
txindex,
|
||||
&indexer.vecs.height_to_first_txindex,
|
||||
&indexes.height_to_txindex_count,
|
||||
exit,
|
||||
)?;
|
||||
} else {
|
||||
let txindex = self.txindex.as_ref().unwrap().as_ref();
|
||||
|
||||
self.height.compute(
|
||||
starting_indexes.height,
|
||||
txindex,
|
||||
&indexer.vecs.height_to_first_txindex,
|
||||
&indexes.height_to_txindex_count,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
self.compute_after_height(indexes, starting_indexes, exit)
|
||||
}
|
||||
|
||||
fn compute_after_height(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.dateindex.from_aligned(
|
||||
starting_indexes.dateindex,
|
||||
&self.height,
|
||||
&indexes.dateindex_to_first_height,
|
||||
&indexes.dateindex_to_height_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.weekindex.from_aligned(
|
||||
starting_indexes.weekindex,
|
||||
&self.dateindex,
|
||||
&indexes.weekindex_to_first_dateindex,
|
||||
&indexes.weekindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.monthindex.from_aligned(
|
||||
starting_indexes.monthindex,
|
||||
&self.dateindex,
|
||||
&indexes.monthindex_to_first_dateindex,
|
||||
&indexes.monthindex_to_dateindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.quarterindex.from_aligned(
|
||||
starting_indexes.quarterindex,
|
||||
&self.monthindex,
|
||||
&indexes.quarterindex_to_first_monthindex,
|
||||
&indexes.quarterindex_to_monthindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.yearindex.from_aligned(
|
||||
starting_indexes.yearindex,
|
||||
&self.monthindex,
|
||||
&indexes.yearindex_to_first_monthindex,
|
||||
&indexes.yearindex_to_monthindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.decadeindex.from_aligned(
|
||||
starting_indexes.decadeindex,
|
||||
&self.yearindex,
|
||||
&indexes.decadeindex_to_first_yearindex,
|
||||
&indexes.decadeindex_to_yearindex_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.difficultyepoch.from_aligned(
|
||||
starting_indexes.difficultyepoch,
|
||||
&self.height,
|
||||
&indexes.difficultyepoch_to_first_height,
|
||||
&indexes.difficultyepoch_to_height_count,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.txindex
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
|
||||
self.height.vecs(),
|
||||
self.dateindex.vecs(),
|
||||
self.weekindex.vecs(),
|
||||
self.difficultyepoch.vecs(),
|
||||
self.monthindex.vecs(),
|
||||
self.quarterindex.vecs(),
|
||||
self.yearindex.vecs(),
|
||||
// self.halvingepoch.vecs(),
|
||||
self.decadeindex.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
||||
impl ComputedVecsFromTxindex<Bitcoin> {
|
||||
pub fn compute_rest_from_sats(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
sats: &ComputedVecsFromTxindex<Sats>,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, Bitcoin>>,
|
||||
) -> Result<()> {
|
||||
let txindex_version = if let Some(txindex) = txindex {
|
||||
txindex.version()
|
||||
} else {
|
||||
self.txindex.as_ref().unwrap().as_ref().version()
|
||||
};
|
||||
|
||||
self.height
|
||||
.validate_computed_version_or_reset_file(txindex_version)?;
|
||||
|
||||
let starting_index = self.height.starting_index(starting_indexes.height);
|
||||
|
||||
(starting_index.unwrap_to_usize()..indexer.vecs.height_to_weight.len())
|
||||
.map(Height::from)
|
||||
.try_for_each(|height| -> Result<()> {
|
||||
if let Some(first) = self.height.first.as_mut() {
|
||||
first.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_first()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(average) = self.height.average.as_mut() {
|
||||
average.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_average()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(sum) = self.height.sum.as_mut() {
|
||||
sum.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_sum()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(max) = self.height.max.as_mut() {
|
||||
max.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_max()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_90p) = self.height._90p.as_mut() {
|
||||
_90p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_90p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_75p) = self.height._75p.as_mut() {
|
||||
_75p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_75p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(median) = self.height.median.as_mut() {
|
||||
median.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_median()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_25p) = self.height._25p.as_mut() {
|
||||
_25p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_25p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_10p) = self.height._10p.as_mut() {
|
||||
_10p.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_10p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(min) = self.height.min.as_mut() {
|
||||
min.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_min()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(last) = self.height.last.as_mut() {
|
||||
last.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_last()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(cumulative) = self.height.cumulative.as_mut() {
|
||||
cumulative.forced_push_at(
|
||||
height,
|
||||
Bitcoin::from(
|
||||
sats.height
|
||||
.unwrap_cumulative()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.height.safe_flush(exit)?;
|
||||
|
||||
self.compute_after_height(indexes, starting_indexes, exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl ComputedVecsFromTxindex<Dollars> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_from_bitcoin(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
bitcoin: &ComputedVecsFromTxindex<Bitcoin>,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, Dollars>>,
|
||||
fetched: &fetched::Vecs,
|
||||
) -> Result<()> {
|
||||
let txindex_version = if let Some(txindex) = txindex {
|
||||
txindex.version()
|
||||
} else {
|
||||
self.txindex.as_ref().unwrap().as_ref().version()
|
||||
};
|
||||
|
||||
self.height
|
||||
.validate_computed_version_or_reset_file(txindex_version)?;
|
||||
|
||||
let starting_index = self.height.starting_index(starting_indexes.height);
|
||||
|
||||
let mut close_iter = fetched.chainindexes_to_close.height.into_iter();
|
||||
|
||||
(starting_index.unwrap_to_usize()..indexer.vecs.height_to_weight.len())
|
||||
.map(Height::from)
|
||||
.try_for_each(|height| -> Result<()> {
|
||||
let price = *close_iter.unwrap_get_inner(height);
|
||||
|
||||
if let Some(first) = self.height.first.as_mut() {
|
||||
first.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_first()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(average) = self.height.average.as_mut() {
|
||||
average.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_average()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(sum) = self.height.sum.as_mut() {
|
||||
sum.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_sum()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(max) = self.height.max.as_mut() {
|
||||
max.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_max()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_90p) = self.height._90p.as_mut() {
|
||||
_90p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_90p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_75p) = self.height._75p.as_mut() {
|
||||
_75p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_75p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(median) = self.height.median.as_mut() {
|
||||
median.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_median()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_25p) = self.height._25p.as_mut() {
|
||||
_25p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_25p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(_10p) = self.height._10p.as_mut() {
|
||||
_10p.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_10p()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(min) = self.height.min.as_mut() {
|
||||
min.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_min()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(last) = self.height.last.as_mut() {
|
||||
last.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_last()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
if let Some(cumulative) = self.height.cumulative.as_mut() {
|
||||
cumulative.forced_push_at(
|
||||
height,
|
||||
price
|
||||
* bitcoin
|
||||
.height
|
||||
.unwrap_cumulative()
|
||||
.into_iter()
|
||||
.unwrap_get_inner(height),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.height.safe_flush(exit)?;
|
||||
|
||||
self.compute_after_height(indexes, starting_indexes, exit)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
mod builder;
|
||||
mod from_dateindex;
|
||||
mod from_height;
|
||||
mod from_height_strict;
|
||||
mod from_txindex;
|
||||
mod ratio_from_dateindex;
|
||||
mod r#type;
|
||||
mod value_from_dateindex;
|
||||
mod value_from_height;
|
||||
mod value_from_txindex;
|
||||
mod value_height;
|
||||
|
||||
pub use builder::*;
|
||||
pub use from_dateindex::*;
|
||||
pub use from_height::*;
|
||||
pub use from_height_strict::*;
|
||||
pub use from_txindex::*;
|
||||
pub use ratio_from_dateindex::*;
|
||||
use r#type::*;
|
||||
pub use value_from_dateindex::*;
|
||||
pub use value_from_height::*;
|
||||
pub use value_from_txindex::*;
|
||||
pub use value_height::*;
|
||||
@@ -0,0 +1,13 @@
|
||||
use std::ops::{Add, Div};
|
||||
|
||||
use brk_vec::StoredType;
|
||||
|
||||
pub trait ComputedType
|
||||
where
|
||||
Self: StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self> + Ord,
|
||||
{
|
||||
}
|
||||
impl<T> ComputedType for T where
|
||||
T: StoredType + From<usize> + Div<usize, Output = Self> + Add<Output = Self> + Ord
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
|
||||
|
||||
use super::StorableVecGeneatorOptions;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedValueVecsFromDateIndex {
|
||||
pub sats: ComputedVecsFromDateIndex<Sats>,
|
||||
pub bitcoin: ComputedVecsFromDateIndex<Bitcoin>,
|
||||
pub dollars: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedValueVecsFromDateIndex {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
compute_dollars: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
name,
|
||||
compute_source,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
bitcoin: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_btc"),
|
||||
true,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_usd"),
|
||||
true,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(
|
||||
&mut EagerVec<DateIndex, Sats>,
|
||||
&Indexer,
|
||||
&indexes::Vecs,
|
||||
&Indexes,
|
||||
&Exit,
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.sats.dateindex.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
|
||||
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, dateindex)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
dateindex: Option<&impl CollectableVec<DateIndex, Sats>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(dateindex) = dateindex {
|
||||
self.sats
|
||||
.compute_rest(indexes, starting_indexes, exit, Some(dateindex))?;
|
||||
|
||||
self.bitcoin.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_sats(starting_indexes.dateindex, dateindex, exit)
|
||||
},
|
||||
)?;
|
||||
} else {
|
||||
let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
|
||||
|
||||
self.sats
|
||||
.compute_rest(indexes, starting_indexes, exit, dateindex)?;
|
||||
|
||||
self.bitcoin.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_sats(
|
||||
starting_indexes.dateindex,
|
||||
self.sats.dateindex.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
let dateindex_to_bitcoin = self.bitcoin.dateindex.as_ref().unwrap();
|
||||
let dateindex_to_close = fetched
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.timeindexes_to_close
|
||||
.dateindex
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
dollars.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_bitcoin(
|
||||
starting_indexes.dateindex,
|
||||
dateindex_to_bitcoin,
|
||||
dateindex_to_close,
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.sats.vecs(),
|
||||
self.bitcoin.vecs(),
|
||||
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
|
||||
use super::{ComputedVecsFromHeight, StorableVecGeneatorOptions};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedValueVecsFromHeight {
|
||||
pub sats: ComputedVecsFromHeight<Sats>,
|
||||
pub bitcoin: ComputedVecsFromHeight<Bitcoin>,
|
||||
pub dollars: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedValueVecsFromHeight {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: StorableVecGeneatorOptions,
|
||||
compute_dollars: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
name,
|
||||
compute_source,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
bitcoin: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_btc"),
|
||||
true,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_usd"),
|
||||
true,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(
|
||||
&mut EagerVec<Height, Sats>,
|
||||
&Indexer,
|
||||
&indexes::Vecs,
|
||||
&Indexes,
|
||||
&Exit,
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.sats.height.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let height: Option<&StoredVec<Height, Sats>> = None;
|
||||
self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, height)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
height: Option<&impl CollectableVec<Height, Sats>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(height) = height {
|
||||
self.sats
|
||||
.compute_rest(indexes, starting_indexes, exit, Some(height))?;
|
||||
|
||||
self.bitcoin.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_sats(starting_indexes.height, height, exit)
|
||||
},
|
||||
)?;
|
||||
} else {
|
||||
let height: Option<&StoredVec<Height, Sats>> = None;
|
||||
|
||||
self.sats
|
||||
.compute_rest(indexes, starting_indexes, exit, height)?;
|
||||
|
||||
self.bitcoin.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_sats(
|
||||
starting_indexes.height,
|
||||
self.sats.height.as_ref().unwrap(),
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
|
||||
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
dollars.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|v, _, _, starting_indexes, exit| {
|
||||
v.compute_from_bitcoin(
|
||||
starting_indexes.height,
|
||||
height_to_bitcoin,
|
||||
height_to_close,
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.sats.vecs(),
|
||||
self.bitcoin.vecs(),
|
||||
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Computation,
|
||||
ComputedVecFrom3, Format, LazyVecFrom1, StoredIndex, StoredVec,
|
||||
};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
|
||||
use super::{ComputedVecsFromTxindex, StorableVecGeneatorOptions};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedValueVecsFromTxindex {
|
||||
pub sats: ComputedVecsFromTxindex<Sats>,
|
||||
pub bitcoin_txindex: LazyVecFrom1<TxIndex, Bitcoin, TxIndex, Sats>,
|
||||
pub bitcoin: ComputedVecsFromTxindex<Bitcoin>,
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub dollars_txindex: Option<
|
||||
ComputedVecFrom3<
|
||||
TxIndex,
|
||||
Dollars,
|
||||
TxIndex,
|
||||
Bitcoin,
|
||||
TxIndex,
|
||||
Height,
|
||||
Height,
|
||||
Close<Dollars>,
|
||||
>,
|
||||
>,
|
||||
pub dollars: Option<ComputedVecsFromTxindex<Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedValueVecsFromTxindex {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
indexes: &indexes::Vecs,
|
||||
source: Option<BoxedAnyIterableVec<TxIndex, Sats>>,
|
||||
version: Version,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
options: StorableVecGeneatorOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let compute_source = source.is_none();
|
||||
let compute_dollars = fetched.is_some();
|
||||
|
||||
let name_in_btc = format!("{name}_in_btc");
|
||||
let name_in_usd = format!("{name}_in_usd");
|
||||
|
||||
let sats = ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
name,
|
||||
compute_source,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
let bitcoin_txindex = LazyVecFrom1::init(
|
||||
&name_in_btc,
|
||||
version + VERSION,
|
||||
source.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s),
|
||||
|txindex: TxIndex, iter| {
|
||||
iter.next_at(txindex.unwrap_to_usize()).map(|(_, value)| {
|
||||
let sats = value.into_owned();
|
||||
Bitcoin::from(sats)
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let bitcoin = ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
&name_in_btc,
|
||||
false,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)?;
|
||||
|
||||
let dollars_txindex = fetched.map(|fetched| {
|
||||
ComputedVecFrom3::forced_import_or_init_from_3(
|
||||
computation,
|
||||
path,
|
||||
&name_in_usd,
|
||||
version + VERSION,
|
||||
format,
|
||||
bitcoin_txindex.boxed_clone(),
|
||||
indexes.txindex_to_height.boxed_clone(),
|
||||
fetched.chainindexes_to_close.height.boxed_clone(),
|
||||
|txindex: TxIndex,
|
||||
txindex_to_btc_iter,
|
||||
txindex_to_height_iter,
|
||||
height_to_close_iter| {
|
||||
let txindex = txindex.unwrap_to_usize();
|
||||
txindex_to_btc_iter.next_at(txindex).and_then(|(_, value)| {
|
||||
let btc = value.into_owned();
|
||||
txindex_to_height_iter
|
||||
.next_at(txindex)
|
||||
.and_then(|(_, value)| {
|
||||
let height = value.into_owned();
|
||||
height_to_close_iter
|
||||
.next_at(height.unwrap_to_usize())
|
||||
.map(|(_, close)| *close.into_owned() * btc)
|
||||
})
|
||||
})
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
sats,
|
||||
bitcoin_txindex,
|
||||
bitcoin,
|
||||
dollars_txindex,
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
&name_in_usd,
|
||||
false,
|
||||
version + VERSION,
|
||||
format,
|
||||
options,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
// pub fn compute_all<F>(
|
||||
// &mut self,
|
||||
// indexer: &Indexer,
|
||||
// indexes: &indexes::Vecs,
|
||||
// fetched: Option<&marketprice::Vecs>,
|
||||
// starting_indexes: &Indexes,
|
||||
// exit: &Exit,
|
||||
// mut compute: F,
|
||||
// ) -> color_eyre::Result<()>
|
||||
// where
|
||||
// F: FnMut(
|
||||
// &mut EagerVec<TxIndex, Sats>,
|
||||
// &Indexer,
|
||||
// &indexes::Vecs,
|
||||
// &Indexes,
|
||||
// &Exit,
|
||||
// ) -> Result<()>,
|
||||
// {
|
||||
// compute(
|
||||
// self.sats.txindex.as_mut().unwrap(),
|
||||
// indexer,
|
||||
// indexes,
|
||||
// starting_indexes,
|
||||
// exit,
|
||||
// )?;
|
||||
|
||||
// let txindex: Option<&StoredVec<TxIndex, Sats>> = None;
|
||||
// self.compute_rest(
|
||||
// indexer,
|
||||
// indexes,
|
||||
// fetched,
|
||||
// starting_indexes,
|
||||
// exit,
|
||||
// txindex,
|
||||
// )?;
|
||||
|
||||
// Ok(())
|
||||
// }
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
txindex: Option<&impl CollectableVec<TxIndex, Sats>>,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(txindex) = txindex {
|
||||
self.sats
|
||||
.compute_rest(indexer, indexes, starting_indexes, exit, Some(txindex))?;
|
||||
} else {
|
||||
let txindex: Option<&StoredVec<TxIndex, Sats>> = None;
|
||||
self.sats
|
||||
.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
|
||||
}
|
||||
|
||||
self.bitcoin.compute_rest_from_sats(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
&self.sats,
|
||||
Some(&self.bitcoin_txindex),
|
||||
)?;
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
let dollars_txindex = self.dollars_txindex.as_mut().unwrap();
|
||||
|
||||
dollars_txindex.compute_if_necessary(
|
||||
starting_indexes.txindex,
|
||||
&indexer.vecs.txindex_to_txid,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
dollars.compute_rest_from_bitcoin(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
&self.bitcoin,
|
||||
Some(dollars_txindex),
|
||||
fetched.as_ref().unwrap(),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.sats.vecs(),
|
||||
vec![&self.bitcoin_txindex as &dyn AnyCollectableVec],
|
||||
self.bitcoin.vecs(),
|
||||
self.dollars_txindex
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
|
||||
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
|
||||
|
||||
use crate::vecs::{Indexes, fetched, indexes};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputedHeightValueVecs {
|
||||
pub sats: Option<EagerVec<Height, Sats>>,
|
||||
pub bitcoin: EagerVec<Height, Bitcoin>,
|
||||
pub dollars: Option<EagerVec<Height, Dollars>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedHeightValueVecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
compute_source: bool,
|
||||
version: Version,
|
||||
format: Format,
|
||||
compute_dollars: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: compute_source.then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
|
||||
.unwrap()
|
||||
}),
|
||||
bitcoin: EagerVec::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_btc"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
&format!("{name}_in_usd"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_all<F>(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
mut compute: F,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
F: FnMut(
|
||||
&mut EagerVec<Height, Sats>,
|
||||
&Indexer,
|
||||
&indexes::Vecs,
|
||||
&Indexes,
|
||||
&Exit,
|
||||
) -> Result<()>,
|
||||
{
|
||||
compute(
|
||||
self.sats.as_mut().unwrap(),
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
let height: Option<&StoredVec<Height, Sats>> = None;
|
||||
self.compute_rest(fetched, starting_indexes, exit, height)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_rest(
|
||||
&mut self,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
height: Option<&impl CollectableVec<Height, Sats>>,
|
||||
) -> color_eyre::Result<()> {
|
||||
if let Some(height) = height {
|
||||
self.bitcoin
|
||||
.compute_from_sats(starting_indexes.height, height, exit)?;
|
||||
} else {
|
||||
self.bitcoin.compute_from_sats(
|
||||
starting_indexes.height,
|
||||
self.sats.as_ref().unwrap(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
let height_to_bitcoin = &self.bitcoin;
|
||||
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
|
||||
|
||||
if let Some(dollars) = self.dollars.as_mut() {
|
||||
dollars.compute_from_bitcoin(
|
||||
starting_indexes.height,
|
||||
height_to_bitcoin,
|
||||
height_to_close,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
vec![&self.bitcoin as &dyn AnyCollectableVec],
|
||||
self.sats.as_ref().map_or(vec![], |v| vec![v]),
|
||||
self.dollars.as_ref().map_or(vec![], |v| vec![v]),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
|
||||
indexes,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex<DifficultyEpoch>,
|
||||
pub indexes_to_halvingepoch: ComputedVecsFromDateIndex<HalvingEpoch>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
"difficulty",
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"difficultyepoch",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
"halvingepoch",
|
||||
true,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
|
||||
self.indexes_to_difficultyepoch.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, indexes, starting_indexes, exit| {
|
||||
let mut height_count_iter = indexes.dateindex_to_height_count.into_iter();
|
||||
vec.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&indexes.dateindex_to_first_height,
|
||||
|(di, height, ..)| {
|
||||
(
|
||||
di,
|
||||
height_to_difficultyepoch_iter.unwrap_get_inner(
|
||||
height + (*height_count_iter.unwrap_get_inner(di) - 1),
|
||||
),
|
||||
)
|
||||
},
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
let mut height_to_halvingepoch_iter = indexes.height_to_halvingepoch.into_iter();
|
||||
self.indexes_to_halvingepoch.compute_all(
|
||||
indexer,
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
|vec, _, indexes, starting_indexes, exit| {
|
||||
let mut height_count_iter = indexes.dateindex_to_height_count.into_iter();
|
||||
vec.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&indexes.dateindex_to_first_height,
|
||||
|(di, height, ..)| {
|
||||
(
|
||||
di,
|
||||
height_to_halvingepoch_iter.unwrap_get_inner(
|
||||
height + (*height_count_iter.unwrap_get_inner(di) - 1),
|
||||
),
|
||||
)
|
||||
},
|
||||
exit,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
self.indexes_to_difficulty.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&indexer.vecs.height_to_difficulty),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.indexes_to_difficulty.vecs(),
|
||||
self.indexes_to_difficultyepoch.vecs(),
|
||||
self.indexes_to_halvingepoch.vecs(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::Version;
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format};
|
||||
|
||||
pub mod blocks;
|
||||
pub mod cointime;
|
||||
pub mod constants;
|
||||
pub mod fetched;
|
||||
pub mod grouped;
|
||||
pub mod indexes;
|
||||
pub mod market;
|
||||
pub mod mining;
|
||||
pub mod stateful;
|
||||
pub mod transactions;
|
||||
|
||||
pub use indexes::Indexes;
|
||||
use log::info;
|
||||
|
||||
use crate::stores::Stores;
|
||||
|
||||
const VERSION: Version = Version::ONE;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub indexes: indexes::Vecs,
|
||||
pub constants: constants::Vecs,
|
||||
pub blocks: blocks::Vecs,
|
||||
pub mining: mining::Vecs,
|
||||
pub market: market::Vecs,
|
||||
pub transactions: transactions::Vecs,
|
||||
pub stateful: stateful::Vecs,
|
||||
pub fetched: Option<fetched::Vecs>,
|
||||
pub cointime: cointime::Vecs,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
indexer: &Indexer,
|
||||
fetch: bool,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let indexes = indexes::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexer,
|
||||
computation,
|
||||
format,
|
||||
)?;
|
||||
|
||||
let fetched = fetch.then(|| {
|
||||
fetched::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
blocks: blocks::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
mining: mining::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
constants: constants::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
market: market::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
)?,
|
||||
stateful: stateful::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
fetched.as_ref(),
|
||||
)?,
|
||||
transactions: transactions::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexer,
|
||||
&indexes,
|
||||
computation,
|
||||
format,
|
||||
fetched.as_ref(),
|
||||
)?,
|
||||
cointime: cointime::Vecs::forced_import(
|
||||
path,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
fetched.as_ref(),
|
||||
)?,
|
||||
indexes,
|
||||
fetched,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
starting_indexes: brk_indexer::Indexes,
|
||||
fetcher: Option<&mut Fetcher>,
|
||||
exit: &Exit,
|
||||
stores: &mut Stores,
|
||||
) -> color_eyre::Result<()> {
|
||||
info!("Computing indexes...");
|
||||
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
|
||||
|
||||
info!("Computing constants...");
|
||||
self.constants
|
||||
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
|
||||
|
||||
info!("Computing blocks...");
|
||||
self.blocks
|
||||
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
|
||||
|
||||
info!("Computing mining...");
|
||||
self.mining
|
||||
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
|
||||
|
||||
if let Some(fetched) = self.fetched.as_mut() {
|
||||
info!("Computing fetched...");
|
||||
fetched.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&starting_indexes,
|
||||
fetcher.unwrap(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
info!("Computing transactions...");
|
||||
self.transactions.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&starting_indexes,
|
||||
self.fetched.as_ref(),
|
||||
exit,
|
||||
)?;
|
||||
|
||||
if let Some(fetched) = self.fetched.as_ref() {
|
||||
info!("Computing market...");
|
||||
self.market.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
fetched,
|
||||
&mut self.transactions,
|
||||
&starting_indexes,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
info!("Computing stateful...");
|
||||
self.stateful.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&self.transactions,
|
||||
self.fetched.as_ref(),
|
||||
&self.market,
|
||||
&mut starting_indexes,
|
||||
exit,
|
||||
stores,
|
||||
)?;
|
||||
|
||||
self.cointime.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&starting_indexes,
|
||||
self.fetched.as_ref(),
|
||||
&self.transactions,
|
||||
&self.stateful,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.constants.vecs(),
|
||||
self.indexes.vecs(),
|
||||
self.blocks.vecs(),
|
||||
self.mining.vecs(),
|
||||
self.market.vecs(),
|
||||
self.transactions.vecs(),
|
||||
self.stateful.vecs(),
|
||||
self.cointime.vecs(),
|
||||
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
use std::{ops::Deref, path::Path};
|
||||
|
||||
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
states::AddressCohortState,
|
||||
vecs::{
|
||||
Indexes, fetched,
|
||||
grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
|
||||
indexes, market,
|
||||
stateful::{
|
||||
common,
|
||||
r#trait::{CohortVecs, DynCohortVecs},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
starting_height: Height,
|
||||
|
||||
pub state: AddressCohortState,
|
||||
|
||||
pub inner: common::Vecs,
|
||||
|
||||
pub height_to_address_count: EagerVec<Height, StoredUsize>,
|
||||
pub indexes_to_address_count: ComputedVecsFromHeight<StoredUsize>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
cohort_name: Option<&str>,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
version: Version,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
states_path: &Path,
|
||||
compute_relative_to_all: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let compute_dollars = fetched.is_some();
|
||||
|
||||
let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}"));
|
||||
|
||||
Ok(Self {
|
||||
starting_height: Height::ZERO,
|
||||
state: AddressCohortState::default_and_import(
|
||||
states_path,
|
||||
cohort_name.unwrap_or_default(),
|
||||
compute_dollars,
|
||||
)?,
|
||||
height_to_address_count: EagerVec::forced_import(
|
||||
path,
|
||||
&suffix("address_count"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
&suffix("address_count"),
|
||||
false,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
StorableVecGeneatorOptions::default().add_last(),
|
||||
)?,
|
||||
inner: common::Vecs::forced_import(
|
||||
path,
|
||||
cohort_name,
|
||||
computation,
|
||||
format,
|
||||
version,
|
||||
fetched,
|
||||
compute_relative_to_all,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl DynCohortVecs for Vecs {
|
||||
fn starting_height(&self) -> Height {
|
||||
[
|
||||
self.state.height().map_or(Height::MAX, |h| h.incremented()),
|
||||
self.height_to_address_count.len().into(),
|
||||
self.inner.starting_height(),
|
||||
]
|
||||
.into_iter()
|
||||
.min()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn init(&mut self, starting_height: Height) {
|
||||
if starting_height > self.starting_height() {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
self.starting_height = starting_height;
|
||||
|
||||
if let Some(prev_height) = starting_height.decremented() {
|
||||
self.state.address_count = *self
|
||||
.height_to_address_count
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height);
|
||||
}
|
||||
|
||||
self.inner
|
||||
.init(&mut self.starting_height, &mut self.state.inner);
|
||||
}
|
||||
|
||||
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
|
||||
self.height_to_address_count
|
||||
.validate_computed_version_or_reset_file(
|
||||
base_version + self.height_to_address_count.inner_version(),
|
||||
)?;
|
||||
|
||||
self.inner.validate_computed_versions(base_version)
|
||||
}
|
||||
|
||||
fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
if self.starting_height > height {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.height_to_address_count.forced_push_at(
|
||||
height,
|
||||
self.state.address_count.into(),
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.inner.forced_pushed_at(height, exit, &self.state.inner)
|
||||
}
|
||||
|
||||
fn compute_then_force_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.inner.compute_then_force_push_unrealized_states(
|
||||
height,
|
||||
height_price,
|
||||
dateindex,
|
||||
date_price,
|
||||
exit,
|
||||
&self.state.inner,
|
||||
)
|
||||
}
|
||||
|
||||
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.height_to_address_count.safe_flush(exit)?;
|
||||
|
||||
self.inner
|
||||
.safe_flush_stateful_vecs(height, exit, &mut self.state.inner)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.indexes_to_address_count.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_address_count),
|
||||
)?;
|
||||
|
||||
self.inner
|
||||
.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
|
||||
}
|
||||
|
||||
fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.inner.vecs(),
|
||||
self.indexes_to_address_count.vecs(),
|
||||
vec![&self.height_to_address_count],
|
||||
]
|
||||
.concat()
|
||||
}
|
||||
}
|
||||
|
||||
impl CohortVecs for Vecs {
|
||||
fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_address_count.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
others
|
||||
.iter()
|
||||
.map(|v| &v.height_to_address_count)
|
||||
.collect::<Vec<_>>()
|
||||
.as_slice(),
|
||||
exit,
|
||||
)?;
|
||||
self.inner.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others.iter().map(|v| &v.inner).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
market: &market::Vecs,
|
||||
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl AnyIterableVec<DateIndex, Bitcoin>,
|
||||
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.inner.compute_rest_part2(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
market,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Vecs {
|
||||
type Target = common::Vecs;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_core::{
|
||||
AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, GroupFilter, Height,
|
||||
Result, Version,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{Computation, Format};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::vecs::{
|
||||
Indexes, fetched,
|
||||
stateful::{
|
||||
address_cohort,
|
||||
r#trait::{CohortVecs, DynCohortVecs},
|
||||
},
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::new(0);
|
||||
|
||||
#[derive(Clone, Deref, DerefMut)]
|
||||
pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>);
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
format: Format,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
states_path: &Path,
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self(
|
||||
AddressGroups {
|
||||
amount_range: ByAmountRange {
|
||||
_0sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_with_0sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1sat_to_10sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1sat_under_10sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10sats_to_100sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10sats_under_100sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100sats_to_1k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_100sats_under_1k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1k_sats_to_10k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1k_sats_under_10k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10k_sats_to_100k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10k_sats_under_100k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100k_sats_to_1m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_100k_sats_under_1m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1m_sats_to_10m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1m_sats_under_10m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10m_sats_to_1btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10m_sats_under_1btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1btc_to_10btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1btc_under_10btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10btc_to_100btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10btc_under_100btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100btc_to_1k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_100btc_under_1k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1k_btc_to_10k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1k_btc_under_10k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10k_btc_to_100k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10k_btc_under_100k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100k_btc_or_more: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_100k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
},
|
||||
lt_amount: ByLowerThanAmount {
|
||||
_10sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_10sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_100sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_1k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_10k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_100k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_1m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_10m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_1btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_10btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_100btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_1k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_10k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_under_100k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
},
|
||||
ge_amount: ByGreatEqualAmount {
|
||||
_1sat: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1sat"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_100sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_100k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_100btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_100btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_1k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_1k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
_10k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
Some("addrs_above_10k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
version + VERSION + Version::ZERO,
|
||||
fetched,
|
||||
states_path,
|
||||
true,
|
||||
)?,
|
||||
},
|
||||
}
|
||||
.into(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn compute_overlapping_vecs(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let by_size_range = self.0.amount_range.as_vec();
|
||||
|
||||
[
|
||||
self.0
|
||||
.ge_amount
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.map(|(filter, vecs)| {
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.into_iter()
|
||||
.filter(|(other, _)| filter.includes(other))
|
||||
.map(|(_, v)| v)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
self.0
|
||||
.lt_amount
|
||||
.as_mut_vec()
|
||||
.into_iter()
|
||||
.map(|(filter, vecs)| {
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.into_iter()
|
||||
.filter(|(other, _)| filter.includes(other))
|
||||
.map(|(_, v)| v)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
]
|
||||
.into_par_iter()
|
||||
.flatten()
|
||||
.try_for_each(|(vecs, stateful)| {
|
||||
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.as_mut_separate_vecs()
|
||||
.par_iter_mut()
|
||||
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
use brk_core::{ByAddressType, Height};
|
||||
use brk_vec::VecIterator;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
|
||||
|
||||
#[derive(Debug, Default, Deref, DerefMut)]
|
||||
pub struct AddressTypeToAddressCount(ByAddressType<usize>);
|
||||
|
||||
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
|
||||
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
|
||||
if let Some(prev_height) = starting_height.decremented() {
|
||||
Self(ByAddressType {
|
||||
p2pk65: groups
|
||||
.p2pk65
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height)
|
||||
.into(),
|
||||
p2pk33: groups
|
||||
.p2pk33
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height)
|
||||
.into(),
|
||||
p2pkh: groups
|
||||
.p2pkh
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height)
|
||||
.into(),
|
||||
p2sh: groups.p2sh.into_iter().unwrap_get_inner(prev_height).into(),
|
||||
p2wpkh: groups
|
||||
.p2wpkh
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height)
|
||||
.into(),
|
||||
p2wsh: groups
|
||||
.p2wsh
|
||||
.into_iter()
|
||||
.unwrap_get_inner(prev_height)
|
||||
.into(),
|
||||
p2tr: groups.p2tr.into_iter().unwrap_get_inner(prev_height).into(),
|
||||
p2a: groups.p2a.into_iter().unwrap_get_inner(prev_height).into(),
|
||||
})
|
||||
} else {
|
||||
Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||