Compare commits

...

131 Commits

Author SHA1 Message Date
nym21 10c16335ed global: opreturn part 2 2026-07-17 21:45:18 +02:00
nym21 29ad37803e global: opreturn 2026-07-16 17:39:05 +02:00
nym21 8d43f340a0 website_next: part 12 2026-07-10 12:45:55 +02:00
nym21 e279116bff website_next: part 11 2026-07-09 23:08:47 +02:00
nym21 5c376acb9f website_next: part 10 2026-07-09 18:49:01 +02:00
nym21 a43b00c12c website_next: part 9 2026-07-08 11:24:52 +02:00
nym21 15fbdc168f website_next: part 8 2026-07-08 11:24:39 +02:00
nym21 046c624abf website_next: part 7 2026-07-06 18:52:51 +02:00
nym21 4d7212fb63 website_next: part 6 2026-07-06 18:05:31 +02:00
nym21 2207ec1b7e website_next: part 5 2026-07-06 15:54:25 +02:00
nym21 3f2a48f248 website_next: part 4 2026-07-06 13:17:11 +02:00
nym21 3f9edb211e website_next: part 3 2026-07-06 11:15:39 +02:00
nym21 eee1a10d2a website_next: part 2 2026-07-05 14:08:57 +02:00
nym21 0bf2cd77dc website_next: snapshot 2026-07-04 22:09:40 +02:00
nym21 a99c06013b website_next: snapshot 2026-07-04 13:27:57 +02:00
nym21 10ef95497f website_next: snapshot 2026-07-03 21:06:32 +02:00
nym21 e0a618837e website_next: snapshot 2026-07-03 11:29:13 +02:00
nym21 5bd1f0b625 deps: bumped 2026-07-02 16:46:47 +02:00
nym21 4d3e8bbcaf website: auto port selection 2026-07-01 17:23:02 +02:00
nym21 153fcdf4e0 website_next: move hash logic to clients 2026-07-01 17:04:29 +02:00
nym21 5d83ee4d70 changelog: updated 2026-06-29 17:02:34 +02:00
nym21 10d29aa2ea clients: bump versions 2026-06-27 18:03:51 +02:00
nym21 528880bd04 release: v0.3.6 2026-06-27 12:18:42 +02:00
nym21 5937fbe724 docs: update generated docs 2026-06-27 12:18:11 +02:00
nym21 7a718293c0 website: fix capture button part 2 2026-06-27 12:12:38 +02:00
nym21 75f37b6695 website: fix capture button 2026-06-27 12:05:39 +02:00
nym21 f9688629ca deps: bumped 2026-06-26 19:40:43 +02:00
nym21 c9e704bdf4 clients: bump versions 2026-06-25 12:31:07 +02:00
nym21 1e0a7d6d0f release: v0.3.5 2026-06-23 18:54:46 +02:00
nym21 565251f12f docs: update generated docs 2026-06-23 18:54:14 +02:00
nym21 9879a986aa fixes 2026-06-23 18:47:51 +02:00
nym21 8d7ef8da61 fixes 2026-06-23 18:42:28 +02:00
nym21 17478a4ac4 website: redesign part 34 2026-06-22 16:42:14 +02:00
nym21 b3031b3375 website: redesign part 33 2026-06-21 17:12:25 +02:00
nym21 2e401379a0 deps: bumped 2026-06-21 17:11:43 +02:00
nym21 45ab6ebf71 website: redesign part 32 2026-06-19 23:16:27 +02:00
nym21 00f7d69ea6 website: redesign part 31 2026-06-18 22:39:28 +02:00
nym21 408d83c350 global: private xpub support part 3 2026-06-17 21:23:26 +02:00
nym21 43df9e098c global: private xpub support part 2 2026-06-17 11:25:42 +02:00
nym21 0c7861071d global: private xpub support part 1 2026-06-16 23:37:03 +02:00
nym21 6f430bdb8c clients: bump versions 2026-06-15 16:24:56 +02:00
nym21 4b415b215d release: v0.3.4 2026-06-14 00:46:39 +02:00
nym21 8614e9eded docs: update generated docs 2026-06-14 00:46:07 +02:00
nym21 c85da92cbc global: add cohorts by entry 2026-06-14 00:40:18 +02:00
nym21 297fc3b855 website: redesign part 30 2026-06-12 12:25:06 +02:00
nym21 c9d5a62fcb website: redesign part 29 2026-06-09 17:01:28 +02:00
nym21 90b3b51c48 website: redesign part 28 2026-06-09 16:12:50 +02:00
nym21 5966ab05e4 website: redesign part 27 2026-06-09 13:35:21 +02:00
nym21 c3506339cd website: redesign part 26 2026-06-09 11:26:19 +02:00
nym21 e54843291e website: redesign part 25 2026-06-09 11:26:14 +02:00
nym21 b0b261fe9f website: redesign part 24 2026-06-08 16:37:53 +02:00
nym21 6786be296d website: redesign part 23 2026-06-08 16:08:52 +02:00
nym21 e5068bbbf3 website: redesign part 24 2026-06-07 23:11:18 +02:00
nym21 36cfe49b20 website: redesign part 23 2026-06-07 16:46:53 +02:00
nym21 33cc13708a website: redesign part 22 2026-06-07 16:11:42 +02:00
nym21 2389632812 website: redesign part 21 2026-06-07 13:10:23 +02:00
nym21 e0bcdb8105 website: redesign part 20 2026-06-07 12:22:49 +02:00
nym21 45e83c98b9 website: redesign part 19 2026-06-07 11:14:15 +02:00
nym21 753bbf3e7e website: redesign part 18 2026-06-07 11:07:15 +02:00
nym21 54cc0cb446 website: redesign part 17 2026-06-07 11:01:31 +02:00
nym21 d64dcb75a9 website: redesign part 16 2026-06-07 01:44:52 +02:00
nym21 f599115f6c website: redesign part 15 2026-06-07 01:38:38 +02:00
nym21 9fc45625ad website: redesign part 14 2026-06-07 01:05:04 +02:00
nym21 c68d1d1fda website: redesign part 13 2026-06-07 00:54:50 +02:00
nym21 6cbe09af23 release: v0.3.3 2026-06-06 22:37:40 +02:00
nym21 96d35d1d29 docs: update generated docs 2026-06-06 22:37:07 +02:00
nym21 e23554811b website: redesign part 12 2026-06-06 22:30:13 +02:00
nym21 041c542046 website: redesign part 11 2026-06-06 22:29:33 +02:00
nym21 66dc7cd8f5 website: redesign part 10 2026-06-06 11:03:55 +02:00
nym21 b00692249c website: redesign part 8 2026-06-05 18:12:46 +02:00
nym21 ff2c04a100 website: redesign part 7 2026-06-05 16:03:04 +02:00
nym21 7cee0e2c5a clients: bump versions 2026-06-04 19:12:58 +02:00
nym21 744032f1f1 release: v0.3.2 2026-06-04 18:56:31 +02:00
nym21 99b171bad6 docs: update generated docs 2026-06-04 18:56:00 +02:00
nym21 37e2b6eae2 changelog: updated 2026-06-04 18:50:35 +02:00
nym21 a967fe8f35 oracle: changes + changelog: updated 2026-06-04 18:35:48 +02:00
nym21 a3f3c54675 oracle: v4 2026-06-04 15:38:01 +02:00
nym21 f41874f438 website: redesign part 6 2026-06-03 18:07:11 +02:00
nym21 98bbfec525 website: redesign part 5 2026-06-03 16:50:52 +02:00
nym21 1bcf3235b6 website: redesign part 4 2026-06-03 16:37:00 +02:00
nym21 07734b8bab website: redesign part 3 2026-06-03 16:26:55 +02:00
nym21 a2fd1e03ad website: redesign part 2 2026-06-03 12:41:26 +02:00
nym21 90e8741fb7 website: redesign part 1 2026-06-03 12:34:05 +02:00
nym21 5f5563fece docs: renamed claude to ai 2026-06-02 12:06:50 +02:00
nym21 c7edfce481 changelog: updated 2026-06-02 09:27:56 +02:00
nym21 7b3dd83b93 clients: bump versions 2026-06-01 20:22:13 +02:00
nym21 cae16227fd release: v0.3.1 2026-06-01 19:19:13 +02:00
nym21 dc2ca0ca27 docs: update generated docs 2026-06-01 19:18:42 +02:00
nym21 d161462137 deps: bumped 2026-06-01 18:10:24 +02:00
nym21 be20633945 heatmaps: part 23 2026-06-01 18:03:41 +02:00
nym21 2bbc535b58 heatmaps: part 22 2026-06-01 17:54:42 +02:00
nym21 88c38e74f9 heatmaps: part 21 2026-06-01 14:25:21 +02:00
nym21 a61b76a4a5 heatmaps: part 20 2026-06-01 13:31:00 +02:00
nym21 46b888337c heatmaps: part 19 2026-06-01 13:20:34 +02:00
nym21 4b49a04186 heatmaps: part 18 2026-06-01 13:03:45 +02:00
nym21 15b0cd2445 heatmaps: part 17 2026-06-01 13:03:39 +02:00
nym21 76720434d7 heatmaps: part 16 2026-06-01 12:19:32 +02:00
nym21 200cd1011e heatmaps: part 15 2026-06-01 12:04:44 +02:00
nym21 cb9f277d49 heatmaps: part 14 2026-06-01 12:01:24 +02:00
nym21 102933b406 heatmaps: part 13 2026-06-01 11:17:00 +02:00
nym21 e64ffac8d1 heatmaps: part 12 2026-06-01 10:56:58 +02:00
nym21 a94d31dfdf heatmaps: part 12 2026-06-01 10:30:44 +02:00
nym21 087a3b6fd6 heatmaps: part 11 2026-06-01 01:04:14 +02:00
nym21 7181d59966 heatmaps: part 10 2026-06-01 00:38:12 +02:00
nym21 3b7734a61a heatmaps: part 9 2026-05-31 23:35:19 +02:00
nym21 7860c5a8bd heatmaps: part 8 2026-05-31 18:57:23 +02:00
nym21 5df399d2f7 heatmaps: part 7 2026-05-31 12:05:48 +02:00
nym21 b2345db279 heatmaps: part 6 2026-05-31 01:38:50 +02:00
nym21 7e2fc8b455 heatmaps: part 5 2026-05-30 15:43:59 +02:00
nym21 c1ff095e4b heatmaps: part 5 2026-05-30 13:16:22 +02:00
nym21 cc8fde59e8 heatmaps: part 4 2026-05-30 11:36:49 +02:00
nym21 e43b53b429 heatmaps: part 3 2026-05-30 11:36:46 +02:00
nym21 6938204a24 heatmaps: part 2 2026-05-29 23:17:39 +02:00
nym21 52883bbdba heatmaps: part 1 2026-05-28 21:58:54 +02:00
nym21 100495fdba deps: bumped 2026-05-27 19:41:37 +02:00
nym21 0ad5be6974 global: snap 2026-05-26 15:33:22 +02:00
nym21 66037c862f global: added support for oracle histograms 2026-05-25 16:44:09 +02:00
nym21 ee20175cbf oracle: cleanup + split lib.rs 2026-05-24 18:40:35 +02:00
nym21 7ad0adf659 oracle: start at 340k 2026-05-24 11:34:57 +02:00
nym21 6219d2301d oracle: snap pre 340k patch 2026-05-24 00:07:25 +02:00
nym21 0aaffc6c43 oracle: doc fixes 2026-05-23 12:18:34 +02:00
nym21 9c74881c5d oracle: snapshot + start at 508k 2026-05-23 00:45:37 +02:00
nym21 bf8de73541 oracle: cleanup 2026-05-22 11:02:56 +02:00
nym21 56e8103178 clients: bump versions 2026-05-22 11:02:12 +02:00
nym21 773c0d090b website: cleanup & fixes 2026-05-22 11:01:34 +02:00
nym21 d6f4c0ac19 changelog: updated 2026-05-22 11:01:07 +02:00
nym21 0552ba60d2 release: v0.3.0 2026-05-18 19:43:54 +02:00
nym21 ff056587f7 docs: update generated docs 2026-05-18 19:43:22 +02:00
nym21 0b871e8600 global: snap 2026-05-18 19:38:05 +02:00
nym21 0bdca9086a docs: moved license 2026-05-18 11:40:28 +02:00
nym21 bbab864ed9 actions: remove release pipeline 2026-05-18 11:38:11 +02:00
1286 changed files with 49138 additions and 300383 deletions
-15
View File
@@ -1,15 +0,0 @@
name: Check outdated dependencies
on:
schedule:
- cron: "0 9 * * 1"
workflow_dispatch:
jobs:
outdated:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- run: cargo install cargo-outdated
- run: cargo outdated --exit-code 1 --depth 1
-296
View File
@@ -1,296 +0,0 @@
# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
#
# Copyright 2022-2024, axodotdev
# SPDX-License-Identifier: MIT or Apache-2.0
#
# CI that:
#
# * checks for a Git Tag that looks like a release
# * builds artifacts with dist (archives, installers, hashes)
# * uploads those artifacts to temporary workflow zip
# * on success, uploads the artifacts to a GitHub Release
#
# Note that the GitHub Release will be created with a generated
# title/body based on your changelogs.
name: Release
permissions:
"contents": "write"
# This task will run whenever you push a git tag that looks like a version
# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
#
# If PACKAGE_NAME is specified, then the announcement will be for that
# package (erroring out if it doesn't have the given version or isn't dist-able).
#
# If PACKAGE_NAME isn't specified, then the announcement will be for all
# (dist-able) packages in the workspace with that version (this mode is
# intended for workspaces with only one dist-able package, or with all dist-able
# packages versioned/released in lockstep).
#
# If you push multiple tags at once, separate instances of this workflow will
# spin up, creating an independent announcement for each one. However, GitHub
# will hard limit this to 3 tags per commit, as it will assume more tags is a
# mistake.
#
# If there's a prerelease-style suffix to the version, then the release(s)
# will be marked as a prerelease.
on:
pull_request:
push:
tags:
- '**[0-9]+.[0-9]+.[0-9]+*'
jobs:
# Run 'dist plan' (or host) to determine what tasks we need to do
plan:
runs-on: "ubuntu-22.04"
outputs:
val: ${{ steps.plan.outputs.manifest }}
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
publishing: ${{ !github.event.pull_request }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install dist
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.2/cargo-dist-installer.sh | sh"
- name: Cache dist
uses: actions/upload-artifact@v4
with:
name: cargo-dist-cache
path: ~/.cargo/bin/dist
# sure would be cool if github gave us proper conditionals...
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
# functionality based on whether this is a pull_request, and whether it's from a fork.
# (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
# but also really annoying to build CI around when it needs secrets to work right.)
- id: plan
run: |
dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
echo "dist ran successfully"
cat plan-dist-manifest.json
echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v4
with:
name: artifacts-plan-dist-manifest
path: plan-dist-manifest.json
# Build and packages all the platform-specific things
build-local-artifacts:
name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
# Let the initial task tell us to not run (currently very blunt)
needs:
- plan
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
strategy:
fail-fast: false
# Target platforms/runners are computed by dist in create-release.
# Each member of the matrix has the following arguments:
#
# - runner: the github runner
# - dist-args: cli flags to pass to dist
# - install-dist: expression to run to install dist on the runner
#
# Typically there will be:
# - 1 "global" task that builds universal installers
# - N "local" tasks that build each platform's binaries and platform-specific installers
matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
runs-on: ${{ matrix.runner }}
container: ${{ matrix.container && matrix.container.image || null }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
steps:
- name: enable windows longpaths
run: |
git config --global core.longpaths true
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install Rust non-interactively if not already installed
if: ${{ matrix.container }}
run: |
if ! command -v cargo > /dev/null 2>&1; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
fi
- name: Install dist
run: ${{ matrix.install_dist.run }}
# Get the dist-manifest
- name: Fetch local artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- name: Install dependencies
run: |
${{ matrix.packages_install }}
- name: Build artifacts
run: |
# Actually do builds and make zips and whatnot
dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
echo "dist ran successfully"
- id: cargo-dist
name: Post-build
# We force bash here just because github makes it really hard to get values up
# to "real" actions without writing to env-vars, and writing to env-vars has
# inconsistent syntax between shell and powershell.
shell: bash
run: |
# Parse out what we just built and upload it to scratch storage
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v4
with:
name: artifacts-build-local-${{ join(matrix.targets, '_') }}
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
# Build and package all the platform-agnostic(ish) things
build-global-artifacts:
needs:
- plan
- build-local-artifacts
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
with:
name: cargo-dist-cache
path: ~/.cargo/bin/
- run: chmod +x ~/.cargo/bin/dist
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
- name: Fetch local artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- id: cargo-dist
shell: bash
run: |
dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
echo "dist ran successfully"
# Parse out what we just built and upload it to scratch storage
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v4
with:
name: artifacts-build-global
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
# Determines if we should publish/announce
host:
needs:
- plan
- build-local-artifacts
- build-global-artifacts
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: "ubuntu-22.04"
outputs:
val: ${{ steps.host.outputs.manifest }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
with:
name: cargo-dist-cache
path: ~/.cargo/bin/
- run: chmod +x ~/.cargo/bin/dist
# Fetch artifacts from scratch-storage
- name: Fetch artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- id: host
shell: bash
run: |
dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
echo "artifacts uploaded and released successfully"
cat dist-manifest.json
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v4
with:
# Overwrite the previous copy
name: artifacts-dist-manifest
path: dist-manifest.json
# Create a GitHub Release while uploading all files to it
- name: "Download GitHub Artifacts"
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: artifacts
merge-multiple: true
- name: Cleanup
run: |
# Remove the granular manifests
rm -f artifacts/*-dist-manifest.json
- name: Create GitHub Release
env:
PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}"
ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}"
RELEASE_COMMIT: "${{ github.sha }}"
run: |
# Write and read notes from a file to avoid quoting breaking things
echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt
gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
announce:
needs:
- plan
- host
# use "always() && ..." to allow us to wait for all publish jobs while
# still allowing individual publish jobs to skip themselves (for prereleases).
# "host" however must run to completion, no skipping allowed!
if: ${{ always() && needs.host.result == 'success' }}
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
+2
View File
@@ -18,6 +18,7 @@ _*
/*.py
/*.json
/*.html
!/btc-cycle-sim.html
/research
/filter_*
/heatmaps*
@@ -25,6 +26,7 @@ _*
/playground
/*.txt
/*.csv
/tmp
# Logs
*.log*
-1
View File
@@ -1 +0,0 @@
Codex will review your output once you are done.
Generated
+294 -448
View File
File diff suppressed because it is too large Load Diff
+36 -44
View File
@@ -4,7 +4,7 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.3.0-beta.11"
package.version = "0.3.6"
package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk"
package.readme = "README.md"
@@ -28,9 +28,6 @@ lto = false
strip = false
inherits = "release"
[profile.dist]
inherits = "release"
[profile.profiling]
inherits = "release"
debug = true
@@ -38,67 +35,62 @@ debug = true
[workspace.dependencies]
aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
bitcoin = { version = "0.32.9", features = ["serde"] }
brk_alloc = { version = "0.3.0-beta.11", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.0-beta.11", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.0-beta.11", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.0-beta.11", path = "crates/brk_cli" }
brk_client = { version = "0.3.0-beta.11", path = "crates/brk_client" }
brk_cohort = { version = "0.3.0-beta.11", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.0-beta.11", path = "crates/brk_computer" }
brk_error = { version = "0.3.0-beta.11", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.0-beta.11", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.0-beta.11", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.0-beta.11", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.0-beta.11", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.0-beta.11", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.0-beta.11", path = "crates/brk_oracle" }
brk_query = { version = "0.3.0-beta.11", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.0-beta.11", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.0-beta.11", path = "crates/brk_rpc" }
brk_server = { version = "0.3.0-beta.11", path = "crates/brk_server" }
brk_store = { version = "0.3.0-beta.11", path = "crates/brk_store" }
brk_traversable = { version = "0.3.0-beta.11", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.0-beta.11", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.0-beta.11", path = "crates/brk_types" }
brk_website = { version = "0.3.0-beta.11", path = "crates/brk_website" }
bitcoin = { version = "0.32.102", features = ["serde"] }
brk_alloc = { version = "0.3.6", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.6", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.6", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.6", path = "crates/brk_cli" }
brk_client = { version = "0.3.6", path = "crates/brk_client" }
brk_cohort = { version = "0.3.6", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.6", path = "crates/brk_computer" }
brk_error = { version = "0.3.6", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.6", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.6", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.6", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.6", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.6", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.6", path = "crates/brk_oracle" }
brk_query = { version = "0.3.6", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.6", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.6", path = "crates/brk_rpc" }
brk_server = { version = "0.3.6", path = "crates/brk_server" }
brk_store = { version = "0.3.6", path = "crates/brk_store" }
brk_traversable = { version = "0.3.6", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.6", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.6", path = "crates/brk_types" }
brk_website = { version = "0.3.6", path = "crates/brk_website" }
byteview = "0.10.1"
color-eyre = "0.6.5"
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
corepc-types = { version = "0.13.0", features = ["std"], default-features = false }
corepc-types = { version = "0.15.0", features = ["std"], default-features = false }
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
fjall = "=3.0.4"
fjall = "3.1.6"
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.24", features = ["perf-inline", "tz-system"], default-features = false }
jiff = { version = "0.2.32", features = ["perf-inline", "tz-system"], default-features = false }
owo-colors = "4.3.0"
parking_lot = "0.12.5"
pco = "1.0.2"
rayon = "1.12.0"
rustc-hash = "2.1.2"
rapidhash = "4.5.1"
rustc-hash = "2.1.3"
schemars = { version = "1.2.1", features = ["indexmap2"] }
serde = "1.0.228"
serde_bytes = "0.11.19"
serde_derive = "1.0.228"
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
smallvec = "1.15.1"
tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
tower-http = { version = "0.6.10", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
serde_json = { version = "1.0.150", features = ["float_roundtrip", "preserve_order"] }
smallvec = "1.15.2"
tokio = { version = "1.52.4", features = ["rt-multi-thread"] }
tower-http = { version = "0.7.0", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tower-layer = "0.3"
tracing = { version = "0.1", default-features = false, features = ["std"] }
ureq = { version = "3.3.0", features = ["json"] }
vecdb = { version = "0.10.3", features = ["derive", "serde_json", "pco", "schemars"] }
vecdb = { version = "0.10.4", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
[workspace.metadata.release]
shared-version = true
tag-name = "v{{version}}"
pre-release-commit-message = "release: v{{version}}"
tag-message = "release: v{{version}}"
allow-branch = ["main", "next"]
[workspace.metadata.dist]
cargo-dist-version = "0.30.2"
ci = "github"
allow-dirty = ["ci"]
installers = []
targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
+1 -1
View File
@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2025 bitcoinresearchkit, kibo.money, satonomics
Copyright (c) 2025 Bitcoin Research Kit
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
+2 -2
View File
@@ -8,5 +8,5 @@ homepage.workspace = true
repository.workspace = true
[dependencies]
libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
mimalloc = { version = "0.1.50" }
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
mimalloc = { version = "0.1.52" }
+4 -3
View File
@@ -6,9 +6,9 @@
use std::collections::BTreeMap;
use brk_cohort::{
AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, CLASS_NAMES, EPOCH_NAMES, LOSS_NAMES, OVER_AGE_NAMES,
OVER_AMOUNT_NAMES, PROFIT_NAMES, PROFITABILITY_RANGE_NAMES, SPENDABLE_TYPE_NAMES, TERM_NAMES,
UNDER_AGE_NAMES, UNDER_AMOUNT_NAMES,
AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, CLASS_NAMES, ENTRY_NAMES, EPOCH_NAMES, LOSS_NAMES,
OVER_AGE_NAMES, OVER_AMOUNT_NAMES, PROFIT_NAMES, PROFITABILITY_RANGE_NAMES,
SPENDABLE_TYPE_NAMES, TERM_NAMES, UNDER_AGE_NAMES, UNDER_AMOUNT_NAMES,
};
use brk_types::{Index, pools};
use serde::Serialize;
@@ -59,6 +59,7 @@ impl CohortConstants {
("TERM_NAMES", to_value(&TERM_NAMES)),
("EPOCH_NAMES", to_value(&EPOCH_NAMES)),
("CLASS_NAMES", to_value(&CLASS_NAMES)),
("ENTRY_NAMES", to_value(&ENTRY_NAMES)),
("SPENDABLE_TYPE_NAMES", to_value(&SPENDABLE_TYPE_NAMES)),
("AGE_RANGE_NAMES", to_value(&AGE_RANGE_NAMES)),
("UNDER_AGE_NAMES", to_value(&UNDER_AGE_NAMES)),
@@ -51,7 +51,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
}
writeln!(
output,
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void }}}} [options]",
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void, cache?: boolean, memCache?: boolean }}}} [options]",
return_type
)
.unwrap();
@@ -60,22 +60,22 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
let params = build_method_params(endpoint);
let params_with_opts = if params.is_empty() {
"{ signal, onValue } = {}".to_string()
"{ signal, onValue, cache, memCache } = {}".to_string()
} else {
format!("{}, {{ signal, onValue }} = {{}}", params)
format!("{}, {{ signal, onValue, cache, memCache }} = {{}}", params)
};
writeln!(output, " async {}({}) {{", method_name, params_with_opts).unwrap();
let path = build_path_template(&endpoint.path, &endpoint.path_params);
let fetch_call: String = if endpoint.returns_binary() {
"this.getBytes(path, { signal, onValue })".to_string()
"this.getBytes(path, { signal, onValue, cache, memCache })".to_string()
} else if endpoint.returns_json() {
"this.getJson(path, { signal, onValue })".to_string()
"this.getJson(path, { signal, onValue, cache, memCache })".to_string()
} else if endpoint.response_kind.text_is_numeric() {
"Number(await this.getText(path, { signal, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
"Number(await this.getText(path, { signal, cache, memCache, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
} else {
"this.getText(path, { signal, onValue })".to_string()
"this.getText(path, { signal, onValue, cache, memCache })".to_string()
};
write_path_assignment(output, endpoint, &path);
@@ -83,7 +83,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
if endpoint.supports_csv {
writeln!(
output,
" if (format === 'csv') return this.getText(path, {{ signal, onValue }});"
" if (format === 'csv') return this.getText(path, {{ signal, onValue, cache, memCache }});"
)
.unwrap();
}
@@ -51,6 +51,12 @@ const _openBrowserCache = (option) => {{
return caches.open(name).catch(() => null);
}};
/**
* @param {{string}} url
* @returns {{URL}}
*/
const _parseBaseUrl = (url) => new URL(url, typeof location === 'undefined' ? undefined : location.href);
/**
* Custom error class for BRK client errors
*/
@@ -237,8 +243,8 @@ function _wrapSeriesData(raw) {{
* @property {{(n: number) => RangeBuilder<T>}} first - Get first n items
* @property {{(n: number) => RangeBuilder<T>}} last - Get last n items
* @property {{(n: number) => SkippedBuilder<T>}} skip - Skip first n items, chain with take()
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch all data
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch all data
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{() => Promise<number>}} len - Get total number of data points
* @property {{() => Promise<Version>}} version - Get the current version of the series
* @property {{Thenable<T>}} then - Thenable (await endpoint)
@@ -253,8 +259,8 @@ function _wrapSeriesData(raw) {{
* @property {{(n: number) => DateRangeBuilder<T>}} first - Get first n items
* @property {{(n: number) => DateRangeBuilder<T>}} last - Get last n items
* @property {{(n: number) => DateSkippedBuilder<T>}} skip - Skip first n items, chain with take()
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch all data
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch all data
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{() => Promise<number>}} len - Get total number of data points
* @property {{() => Promise<Version>}} version - Get the current version of the series
* @property {{DateThenable<T>}} then - Thenable (await endpoint)
@@ -263,41 +269,53 @@ function _wrapSeriesData(raw) {{
/** @typedef {{SeriesEndpoint<any>}} AnySeriesEndpoint */
/**
* @template T
* @typedef {{Object}} ClientFetchOptions
* @property {{AbortSignal}} [signal] - Abort this request
* @property {{boolean}} [cache] - Use HTTP/browser/client caches. Set false for a no-store network fetch.
* @property {{boolean}} [memCache] - Use the parsed in-memory response cache. Set false for large one-shot reads.
* @property {{(value: T) => void}} [onValue] - Receive stale/fresh values as they arrive
*/
/** @template T @typedef {{ClientFetchOptions<SeriesData<T>> | ((value: SeriesData<T>) => void)}} SeriesFetchArg */
/** @template T @typedef {{ClientFetchOptions<DateSeriesData<T>> | ((value: DateSeriesData<T>) => void)}} DateSeriesFetchArg */
/** @template T @typedef {{Object}} SingleItemBuilder
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the item
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch the item
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{Thenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} DateSingleItemBuilder
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the item
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch the item
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{DateThenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} SkippedBuilder
* @property {{(n: number) => RangeBuilder<T>}} take - Take n items after skipped position
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{Thenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} DateSkippedBuilder
* @property {{(n: number) => DateRangeBuilder<T>}} take - Take n items after skipped position
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{DateThenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} RangeBuilder
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the range
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch the range
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{Thenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} DateRangeBuilder
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the range
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch the range
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{DateThenable<T>}} then - Thenable
*/
@@ -344,8 +362,8 @@ function _endpoint(client, name, index) {{
* @returns {{DateRangeBuilder<T>}}
*/
const rangeBuilder = (start, end) => ({{
fetch(onValue) {{ return client._fetchSeriesData(buildPath(start, end), onValue); }},
fetchCsv() {{ return client.getText(buildPath(start, end, 'csv')); }},
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(start, end), arg, options); }},
fetchCsv(options) {{ return client.getText(buildPath(start, end, 'csv'), options); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
}});
@@ -354,8 +372,8 @@ function _endpoint(client, name, index) {{
* @returns {{DateSingleItemBuilder<T>}}
*/
const singleItemBuilder = (idx) => ({{
fetch(onValue) {{ return client._fetchSeriesData(buildPath(idx, idx + 1), onValue); }},
fetchCsv() {{ return client.getText(buildPath(idx, idx + 1, 'csv')); }},
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(idx, idx + 1), arg, options); }},
fetchCsv(options) {{ return client.getText(buildPath(idx, idx + 1, 'csv'), options); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
}});
@@ -365,8 +383,8 @@ function _endpoint(client, name, index) {{
*/
const skippedBuilder = (start) => ({{
take(n) {{ return rangeBuilder(start, start + n); }},
fetch(onValue) {{ return client._fetchSeriesData(buildPath(start, undefined), onValue); }},
fetchCsv() {{ return client.getText(buildPath(start, undefined, 'csv')); }},
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(start, undefined), arg, options); }},
fetchCsv(options) {{ return client.getText(buildPath(start, undefined, 'csv'), options); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
}});
@@ -381,8 +399,8 @@ function _endpoint(client, name, index) {{
first(n) {{ return rangeBuilder(undefined, n); }},
last(n) {{ return n === 0 ? rangeBuilder(undefined, 0) : rangeBuilder(-n, undefined); }},
skip(n) {{ return skippedBuilder(n); }},
fetch(onValue) {{ return client._fetchSeriesData(buildPath(), onValue); }},
fetchCsv() {{ return client.getText(buildPath(undefined, undefined, 'csv')); }},
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(), arg, options); }},
fetchCsv(options) {{ return client.getText(buildPath(undefined, undefined, 'csv'), options); }},
len() {{ return client.getSeriesLen(name, index); }},
version() {{ return client.getSeriesVersion(name, index); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
@@ -403,6 +421,9 @@ class BrkClientBase {{
const isString = typeof options === 'string';
const rawUrl = isString ? options : options.baseUrl;
this.baseUrl = rawUrl.endsWith('/') ? rawUrl.slice(0, -1) : rawUrl;
const url = _parseBaseUrl(this.baseUrl);
this.url = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
this.domain = url.hostname;
this.timeout = isString ? 5000 : (options.timeout ?? 5000);
/** @type {{Promise<Cache | null>}} */
this._browserCachePromise = _openBrowserCache(isString ? undefined : options.browserCache);
@@ -448,14 +469,17 @@ class BrkClientBase {{
/**
* @param {{string}} path
* @param {{{{ signal?: AbortSignal }}}} [options]
* @param {{{{ signal?: AbortSignal, cache?: boolean }}}} [options]
* @returns {{Promise<Response>}}
*/
async get(path, {{ signal }} = {{}}) {{
async get(path, {{ signal, cache = true }} = {{}}) {{
const url = `${{this.baseUrl}}${{path}}`;
const signals = [AbortSignal.timeout(this.timeout)];
if (signal) signals.push(signal);
const res = await fetch(url, {{ signal: AbortSignal.any(signals) }});
/** @type {{RequestInit}} */
const init = {{ signal: AbortSignal.any(signals) }};
if (!cache) init.cache = 'no-store';
const res = await fetch(url, init);
if (!res.ok) throw new BrkError(`HTTP ${{res.status}}: ${{url}}`, res.status);
return res;
}}
@@ -475,14 +499,22 @@ class BrkClientBase {{
* @template T
* @param {{string}} path
* @param {{(res: Response) => Promise<T>}} parse - Response body reader
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @param {{ClientFetchOptions<T>}} [options]
* @returns {{Promise<T>}}
*/
async _getCached(path, parse, {{ onValue, signal }} = {{}}) {{
async _getCached(path, parse, {{ onValue, signal, cache = true, memCache = true }} = {{}}) {{
if (!cache) {{
const res = await this.get(path, {{ signal, cache }});
const value = await parse(res);
if (onValue) onValue(value);
return value;
}}
const url = `${{this.baseUrl}}${{path}}`;
const useMemCache = memCache !== false;
/** @type {{_MemEntry<T> | undefined}} */
const memHit = this._memGet(url);
const browserCache = this._browserCache ?? await this._browserCachePromise;
const memHit = useMemCache ? this._memGet(url) : undefined;
const browserCache = this._browserCache;
// L1 fast path: deliver from memCache, revalidate via network.
// ETag match → zero parse, zero clone, zero cache write, no second onValue fire.
@@ -494,11 +526,11 @@ class BrkClientBase {{
if (netEtag && netEtag === memHit.etag) return memHit.value;
const cloned = browserCache ? res.clone() : null;
const value = await parse(res);
this._memSet(url, netEtag, value);
if (useMemCache) this._memSet(url, netEtag, value);
if (onValue) onValue(value);
if (cloned && browserCache) {{
const cache = browserCache;
_runIdle(() => cache.put(url, cloned));
const cacheStore = browserCache;
_runIdle(() => cacheStore.put(url, cloned));
}}
return value;
}} catch {{
@@ -513,7 +545,7 @@ class BrkClientBase {{
if (!res || networkSettled) return null;
const value = await parse(res);
if (networkSettled) return value;
this._memSet(url, res.headers.get('ETag'), value);
if (useMemCache) this._memSet(url, res.headers.get('ETag'), value);
onValue(value);
return value;
}}).catch(() => null)
@@ -524,15 +556,15 @@ class BrkClientBase {{
networkSettled = true;
const netEtag = res.headers.get('ETag');
// Stale won and populated memCache with matching ETag → reuse, skip parse + second onValue.
const populated = /** @type {{_MemEntry<T> | undefined}} */ (this._memGet(url));
const populated = useMemCache ? /** @type {{_MemEntry<T> | undefined}} */ (this._memGet(url)) : undefined;
if (populated && netEtag && netEtag === populated.etag) return populated.value;
const cloned = browserCache ? res.clone() : null;
const value = await parse(res);
this._memSet(url, netEtag, value);
if (useMemCache) this._memSet(url, netEtag, value);
if (onValue) onValue(value);
if (cloned && browserCache) {{
const cache = browserCache;
_runIdle(() => cache.put(url, cloned));
const cacheStore = browserCache;
_runIdle(() => cacheStore.put(url, cloned));
}}
return value;
}} catch (e) {{
@@ -546,7 +578,7 @@ class BrkClientBase {{
* Make a GET request expecting a JSON response. Cached and supports `onValue`.
* @template T
* @param {{string}} path
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @param {{ClientFetchOptions<T>}} [options]
* @returns {{Promise<T>}}
*/
getJson(path, options) {{
@@ -557,7 +589,7 @@ class BrkClientBase {{
* Make a GET request expecting a text response (text/plain, text/csv, ...).
* Cached and supports `onValue`, same as `getJson`.
* @param {{string}} path
* @param {{{{ onValue?: (value: string) => void, signal?: AbortSignal }}}} [options]
* @param {{ClientFetchOptions<string>}} [options]
* @returns {{Promise<string>}}
*/
getText(path, options) {{
@@ -568,7 +600,7 @@ class BrkClientBase {{
* Make a GET request expecting binary data (application/octet-stream).
* Cached and supports `onValue`, same as `getJson`.
* @param {{string}} path
* @param {{{{ onValue?: (value: Uint8Array) => void, signal?: AbortSignal }}}} [options]
* @param {{ClientFetchOptions<Uint8Array>}} [options]
* @returns {{Promise<Uint8Array>}}
*/
getBytes(path, options) {{
@@ -640,12 +672,17 @@ class BrkClientBase {{
* Fetch series data and wrap with helper methods (internal)
* @template T
* @param {{string}} path
* @param {{(value: DateSeriesData<T>) => void}} [onValue]
* @param {{DateSeriesFetchArg<T>}} [arg]
* @param {{ClientFetchOptions<DateSeriesData<T>>}} [options]
* @returns {{Promise<DateSeriesData<T>>}}
*/
async _fetchSeriesData(path, onValue) {{
async _fetchSeriesData(path, arg, options) {{
const requestOptions = typeof arg === 'function'
? {{ ...(options ?? {{}}), onValue: arg }}
: {{ ...(arg ?? {{}}), ...(options ?? {{}}) }};
const onValue = requestOptions.onValue;
const wrappedOnValue = onValue ? (/** @type {{SeriesData<T>}} */ raw) => onValue(_wrapSeriesData(raw)) : undefined;
const raw = await this.getJson(path, {{ onValue: wrappedOnValue }});
const raw = await this.getJson(path, {{ ...requestOptions, onValue: wrappedOnValue }});
return _wrapSeriesData(raw);
}}
}}
@@ -669,6 +706,160 @@ const _p = (prefix, acc) => acc ? `${{prefix}}_${{acc}}` : prefix;
"#
)
.unwrap();
output.push_str(r##"
const _MASK_64 = 0xffffffffffffffffn;
const _RAPIDHASH_SECRETS = /** @type {const} */ ([
0x2d358dccaa6c78a5n,
0x8bb84b93962eacc9n,
0x4b33a62ed433d4a3n,
0x4d5a2da51de1aa47n,
0xa0761d6478bd642fn,
0xe7037ed1a0b428dbn,
0x90ed1765281c388cn,
]);
const _RAPIDHASH_SEED = _rapidHashSeed(0n);
/** @param {bigint} value */
function _u64(value) {
return value & _MASK_64;
}
/** @param {bigint} left @param {bigint} right */
function _rapidMix(left, right) {
const result = _u64(left) * _u64(right);
return _u64(result) ^ _u64(result >> 64n);
}
/** @param {bigint} left @param {bigint} right @returns {[bigint, bigint]} */
function _rapidMum(left, right) {
const result = _u64(left) * _u64(right);
return [_u64(result), _u64(result >> 64n)];
}
/** @param {bigint} seed */
function _rapidHashSeed(seed) {
return _u64(seed ^ _rapidMix(seed ^ _RAPIDHASH_SECRETS[2], _RAPIDHASH_SECRETS[1]));
}
/** @param {Uint8Array} bytes @param {number} offset */
function _readU32(bytes, offset) {
return (
BigInt(bytes[offset]) |
(BigInt(bytes[offset + 1]) << 8n) |
(BigInt(bytes[offset + 2]) << 16n) |
(BigInt(bytes[offset + 3]) << 24n)
);
}
/** @param {Uint8Array} bytes @param {number} offset */
function _readU64(bytes, offset) {
return _readU32(bytes, offset) | (_readU32(bytes, offset + 4) << 32n);
}
/** @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload */
function _asUint8Array(payload) {
if (payload instanceof Uint8Array) return payload;
if (payload instanceof ArrayBuffer) return new Uint8Array(payload);
if (ArrayBuffer.isView(payload)) return new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength);
if (Array.isArray(payload)) return new Uint8Array(payload);
throw new Error("Expected address payload bytes");
}
/** @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload */
function _rapidHashV3(payload) {
const bytes = _asUint8Array(payload);
const length = bytes.length;
if (length === 0) throw new Error("Expected a non-empty address payload");
if (length > 65) throw new Error("Expected at most 65 address payload bytes");
let seed = _RAPIDHASH_SEED;
let a = 0n;
let b = 0n;
let remainder;
if (length <= 16) {
if (length >= 4) {
seed ^= BigInt(length);
if (length >= 8) {
a ^= _readU64(bytes, 0);
b ^= _readU64(bytes, length - 8);
} else {
a ^= _readU32(bytes, 0);
b ^= _readU32(bytes, length - 4);
}
} else if (length > 0) {
a ^= (BigInt(bytes[0]) << 45n) | BigInt(bytes[length - 1]);
b ^= BigInt(bytes[length >> 1]);
}
remainder = BigInt(length);
} else {
seed = _rapidMix(_readU64(bytes, 0) ^ _RAPIDHASH_SECRETS[2], _readU64(bytes, 8) ^ seed);
if (length > 32) {
seed = _rapidMix(_readU64(bytes, 16) ^ _RAPIDHASH_SECRETS[2], _readU64(bytes, 24) ^ seed);
if (length > 48) {
seed = _rapidMix(_readU64(bytes, 32) ^ _RAPIDHASH_SECRETS[1], _readU64(bytes, 40) ^ seed);
if (length > 64) {
seed = _rapidMix(_readU64(bytes, 48) ^ _RAPIDHASH_SECRETS[1], _readU64(bytes, 56) ^ seed);
}
}
}
remainder = BigInt(length);
a ^= _readU64(bytes, length - 16) ^ remainder;
b ^= _readU64(bytes, length - 8);
}
a ^= _RAPIDHASH_SECRETS[1];
b ^= seed;
[a, b] = _rapidMum(a, b);
return _rapidMix(a ^ 0xaaaaaaaaaaaaaaaan, b ^ _RAPIDHASH_SECRETS[1] ^ remainder);
}
/** @param {number} nibbles */
function _validateHashPrefixNibbles(nibbles) {
if (!Number.isInteger(nibbles) || nibbles < 1 || nibbles > 16) {
throw new Error("Expected hash-prefix length from 1 to 16 hex nibbles");
}
}
/** @param {OutputType} addrType @returns {number[]} */
function _addressPayloadLengths(addrType) {
switch (addrType) {
case "p2a": return [2];
case "p2pk": return [33, 65];
case "p2pkh":
case "p2sh":
case "v0_p2wpkh": return [20];
case "v0_p2wsh":
case "v1_p2tr": return [32];
default:
throw new Error(`Unsupported address type for address payload hash-prefix: ${addrType}`);
}
}
/**
* @param {OutputType} addrType
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload
*/
function _validateAddressPayloadForType(addrType, payload) {
const length = _asUint8Array(payload).length;
const expected = _addressPayloadLengths(addrType);
if (!expected.includes(length)) {
throw new Error(`Expected ${addrType} address payload length ${expected.join(" or ")} bytes`);
}
}
/**
* Compute the RapidHash v3 hash-prefix used by `/api/address/hash-prefix/{addr_type}/{prefix}`.
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload - Raw address payload bytes
* @param {number} nibbles - Prefix length from 1 to 16 hex nibbles
* @returns {string}
*/
function addressPayloadHashPrefix(payload, nibbles) {
_validateHashPrefixNibbles(nibbles);
return _rapidHashV3(payload).toString(16).padStart(16, "0").slice(0, nibbles);
}
"##);
}
/// Generate static constants for the BrkClient class.
@@ -111,6 +111,32 @@ pub fn generate_main_client(
writeln!(output, " this.series = this._buildTree();").unwrap();
writeln!(output, " }}\n").unwrap();
output.push_str(r##" /**
* Compute the RapidHash v3 hash-prefix for raw address payload bytes.
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload
* @param {number} nibbles
* @returns {string}
*/
static addressPayloadHashPrefix(payload, nibbles) {
return addressPayloadHashPrefix(payload, nibbles);
}
/**
* Fetch address hash-prefix matches from raw address payload bytes.
* @param {OutputType} addrType
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload - Raw payload bytes matching addrType length
* @param {number} nibbles
* @param {{ signal?: AbortSignal, onValue?: (value: AddrHashPrefixMatches) => void, cache?: boolean, memCache?: boolean }} [options]
* @returns {Promise<AddrHashPrefixMatches>}
*/
getAddressPayloadHashPrefixMatches(addrType, payload, nibbles, options = {}) {
_validateAddressPayloadForType(addrType, payload);
const prefix = addressPayloadHashPrefix(payload, nibbles);
return this.getAddressHashPrefixMatches(addrType, prefix, options);
}
"##);
writeln!(output, " /**").unwrap();
writeln!(output, " * @private").unwrap();
writeln!(output, " * @returns {{SeriesTree}}").unwrap();
@@ -161,7 +187,11 @@ pub fn generate_main_client(
writeln!(output, "}}\n").unwrap();
writeln!(output, "export {{ BrkClient, BrkError }};").unwrap();
writeln!(
output,
"export {{ BrkClient, BrkError, addressPayloadHashPrefix }};"
)
.unwrap();
}
#[allow(clippy::too_many_arguments)]
@@ -84,6 +84,23 @@ pub fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
.unwrap();
writeln!(output, " return _date_to_index(index, d)").unwrap();
writeln!(output).unwrap();
output.push_str(r#" @staticmethod
def address_payload_hash_prefix(payload: Union[bytes, bytearray, memoryview], nibbles: int) -> str:
"""Compute the RapidHash v3 hash-prefix for raw address payload bytes."""
return address_payload_hash_prefix(payload, nibbles)
def get_address_payload_hash_prefix_matches(
self,
addr_type: OutputType,
payload: Union[bytes, bytearray, memoryview],
nibbles: int,
) -> AddrHashPrefixMatches:
"""Fetch address hash-prefix matches from raw payload bytes matching addr_type length."""
_validate_address_payload_for_type(addr_type, payload)
return self.get_address_hash_prefix_matches(addr_type, address_payload_hash_prefix(payload, nibbles))
"#);
// Generate API methods
generate_api_methods(output, endpoints);
}
@@ -147,6 +147,125 @@ def _p(prefix: str, acc: str) -> str:
"#
)
.unwrap();
output.push_str(r#"
_MASK_64 = (1 << 64) - 1
_RAPIDHASH_SECRETS = (
0x2d358dccaa6c78a5,
0x8bb84b93962eacc9,
0x4b33a62ed433d4a3,
0x4d5a2da51de1aa47,
0xa0761d6478bd642f,
0xe7037ed1a0b428db,
0x90ed1765281c388c,
)
_RAPIDHASH_SEED = 0
def _u64(value: int) -> int:
return value & _MASK_64
def _rapid_mix(left: int, right: int) -> int:
result = _u64(left) * _u64(right)
return _u64(result) ^ _u64(result >> 64)
def _rapid_mum(left: int, right: int) -> Tuple[int, int]:
result = _u64(left) * _u64(right)
return _u64(result), _u64(result >> 64)
def _rapid_hash_seed(seed: int) -> int:
return _u64(seed ^ _rapid_mix(seed ^ _RAPIDHASH_SECRETS[2], _RAPIDHASH_SECRETS[1]))
_RAPIDHASH_SEED = _rapid_hash_seed(0)
def _read_u32(data: bytes, offset: int) -> int:
return int.from_bytes(data[offset:offset + 4], "little")
def _read_u64(data: bytes, offset: int) -> int:
return int.from_bytes(data[offset:offset + 8], "little")
def _rapid_hash_v3(payload: Union[bytes, bytearray, memoryview]) -> int:
data = bytes(payload)
length = len(data)
if length == 0:
raise ValueError("Expected a non-empty address payload")
if length > 65:
raise ValueError("Expected at most 65 address payload bytes")
seed = _RAPIDHASH_SEED
a = 0
b = 0
if length <= 16:
if length >= 4:
seed ^= length
if length >= 8:
a ^= _read_u64(data, 0)
b ^= _read_u64(data, length - 8)
else:
a ^= _read_u32(data, 0)
b ^= _read_u32(data, length - 4)
elif length > 0:
a ^= (data[0] << 45) | data[length - 1]
b ^= data[length >> 1]
remainder = length
else:
if length > 16:
seed = _rapid_mix(_read_u64(data, 0) ^ _RAPIDHASH_SECRETS[2], _read_u64(data, 8) ^ seed)
if length > 32:
seed = _rapid_mix(_read_u64(data, 16) ^ _RAPIDHASH_SECRETS[2], _read_u64(data, 24) ^ seed)
if length > 48:
seed = _rapid_mix(_read_u64(data, 32) ^ _RAPIDHASH_SECRETS[1], _read_u64(data, 40) ^ seed)
if length > 64:
seed = _rapid_mix(_read_u64(data, 48) ^ _RAPIDHASH_SECRETS[1], _read_u64(data, 56) ^ seed)
remainder = length
a ^= _read_u64(data, length - 16) ^ remainder
b ^= _read_u64(data, length - 8)
a ^= _RAPIDHASH_SECRETS[1]
b ^= seed
a, b = _rapid_mum(a, b)
return _rapid_mix(a ^ 0xaaaaaaaaaaaaaaaa, b ^ _RAPIDHASH_SECRETS[1] ^ remainder)
def _validate_hash_prefix_nibbles(nibbles: int) -> None:
if isinstance(nibbles, bool) or not isinstance(nibbles, int) or nibbles < 1 or nibbles > 16:
raise ValueError("Expected hash-prefix length from 1 to 16 hex nibbles")
def _address_payload_lengths(addr_type: OutputType) -> Tuple[int, ...]:
if addr_type == "p2a":
return (2,)
if addr_type == "p2pk":
return (33, 65)
if addr_type in ("p2pkh", "p2sh", "v0_p2wpkh"):
return (20,)
if addr_type in ("v0_p2wsh", "v1_p2tr"):
return (32,)
raise ValueError(f"Unsupported address type for address payload hash-prefix: {addr_type}")
def _validate_address_payload_for_type(addr_type: OutputType, payload: Union[bytes, bytearray, memoryview]) -> None:
length = len(bytes(payload))
expected = _address_payload_lengths(addr_type)
if length not in expected:
joined = " or ".join(str(value) for value in expected)
raise ValueError(f"Expected {addr_type} address payload length {joined} bytes")
def address_payload_hash_prefix(payload: Union[bytes, bytearray, memoryview], nibbles: int) -> str:
"""Compute the RapidHash v3 hash-prefix used by `/api/address/hash-prefix/{addr_type}/{prefix}`."""
_validate_hash_prefix_nibbles(nibbles)
return f"{_rapid_hash_v3(payload):016x}"[:nibbles]
"#);
}
/// Generate the SeriesData and SeriesEndpoint classes
@@ -76,6 +76,36 @@ impl BrkClient {{
)
.unwrap();
output.push_str(r#" /// Decode a mainnet Bitcoin address into the BRK address type and raw payload bytes.
pub fn decode_address_payload(address: &str) -> Result<AddressPayload> {
decode_address_payload(address)
}
/// Compute the RapidHash v3 hash-prefix for raw address payload bytes.
pub fn address_payload_hash_prefix(payload: &[u8], nibbles: usize) -> Result<String> {
address_payload_hash_prefix(payload, nibbles)
}
/// Decode a mainnet Bitcoin address and compute its hash prefix.
pub fn address_hash_prefix(address: &str, nibbles: usize) -> Result<AddressHashPrefix> {
address_hash_prefix(address, nibbles)
}
/// Fetch address hash-prefix matches from raw payload bytes matching `addr_type` length.
pub fn get_address_payload_hash_prefix_matches(&self, addr_type: OutputType, payload: &[u8], nibbles: usize) -> Result<AddrHashPrefixMatches> {
validate_address_payload_for_type(addr_type, payload)?;
let prefix = address_payload_hash_prefix(payload, nibbles)?;
self.get_address_hash_prefix_matches(addr_type, &prefix)
}
/// Fetch address hash-prefix matches for a mainnet Bitcoin address.
pub fn get_address_hash_prefix_matches_for_address(&self, address: &str, nibbles: usize) -> Result<AddrHashPrefixMatches> {
let hashed = address_hash_prefix(address, nibbles)?;
self.get_address_hash_prefix_matches(hashed.addr_type, &hashed.prefix)
}
"#);
generate_api_methods(output, endpoints);
writeln!(output, "}}").unwrap();
@@ -118,6 +148,23 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
"get_text"
};
if endpoint.path == "/api/address/hash-prefix/{addr_type}/{prefix}" {
writeln!(
output,
" let addr_type = address_payload_type_path(addr_type)?;"
)
.unwrap();
writeln!(
output,
" self.base.{}(&format!(\"{}\"{}))",
fetch_method, path, index_arg
)
.unwrap();
writeln!(output, " }}").unwrap();
writeln!(output).unwrap();
return;
}
if endpoint.query_params.is_empty() {
writeln!(
output,
@@ -11,7 +11,8 @@ use crate::{
pub fn generate_imports(output: &mut String) {
writeln!(
output,
r#"use std::sync::Arc;
r#"use std::str::FromStr;
use std::sync::Arc;
use std::ops::{{Bound, RangeBounds}};
use serde::de::DeserializeOwned;
pub use brk_cohort::*;
@@ -43,6 +44,97 @@ impl std::error::Error for BrkError {{}}
/// Result type for BRK client operations.
pub type Result<T> = std::result::Result<T, BrkError>;
/// BRK address type and raw payload bytes used by the hash-prefix index.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AddressPayload {{
pub addr_type: OutputType,
pub payload: Vec<u8>,
}}
/// BRK address type and leading hex nibbles of the address-payload hash.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AddressHashPrefix {{
pub addr_type: OutputType,
pub prefix: String,
}}
/// Compute the RapidHash v3 hash-prefix used by `/api/address/hash-prefix/{{addr_type}}/{{prefix}}`.
pub fn address_payload_hash_prefix(payload: &[u8], nibbles: usize) -> Result<String> {{
if payload.is_empty() {{
return Err(BrkError {{ message: "Expected a non-empty address payload".to_string() }});
}}
if payload.len() > 65 {{
return Err(BrkError {{ message: "Expected at most 65 address payload bytes".to_string() }});
}}
if !(1..=16).contains(&nibbles) {{
return Err(BrkError {{ message: "Expected hash-prefix length from 1 to 16 hex nibbles".to_string() }});
}}
Ok(format!("{{:016x}}", rapidhash::v3::rapidhash_v3(payload))[..nibbles].to_string())
}}
fn validate_address_payload_for_type(addr_type: OutputType, payload: &[u8]) -> Result<()> {{
let expected: &[usize] = match addr_type {{
OutputType::P2A => &[2],
OutputType::P2PK33 => &[33],
OutputType::P2PK65 => &[65],
OutputType::P2PKH | OutputType::P2SH | OutputType::P2WPKH => &[20],
OutputType::P2WSH | OutputType::P2TR => &[32],
OutputType::P2MS | OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => {{
return Err(BrkError {{ message: format!("Unsupported address type for address payload hash-prefix: {{addr_type:?}}") }});
}},
}};
let addr_type = address_payload_type_path(addr_type)?;
if !expected.contains(&payload.len()) {{
let joined = expected
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(" or ");
return Err(BrkError {{ message: format!("Expected {{addr_type}} address payload length {{joined}} bytes") }});
}}
Ok(())
}}
fn address_payload_type_path(addr_type: OutputType) -> Result<&'static str> {{
match addr_type {{
OutputType::P2A => Ok("p2a"),
OutputType::P2PK33 | OutputType::P2PK65 => Ok("p2pk"),
OutputType::P2PKH => Ok("p2pkh"),
OutputType::P2SH => Ok("p2sh"),
OutputType::P2WPKH => Ok("v0_p2wpkh"),
OutputType::P2WSH => Ok("v0_p2wsh"),
OutputType::P2TR => Ok("v1_p2tr"),
OutputType::P2MS | OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => {{
Err(BrkError {{ message: format!("Unsupported address type for address payload hash-prefix: {{addr_type:?}}") }})
}},
}}
}}
/// Decode a mainnet Bitcoin address into the BRK address type and raw payload bytes.
pub fn decode_address_payload(address: &str) -> Result<AddressPayload> {{
if address.is_empty() {{
return Err(BrkError {{ message: "Expected an address string".to_string() }});
}}
let addr_bytes = AddrBytes::from_str(address).map_err(|e| BrkError {{ message: e.to_string() }})?;
let addr_type = OutputType::from(&addr_bytes);
Ok(AddressPayload {{
addr_type,
payload: addr_bytes.as_slice().to_vec(),
}})
}}
/// Decode a mainnet Bitcoin address and compute its hash prefix.
pub fn address_hash_prefix(address: &str, nibbles: usize) -> Result<AddressHashPrefix> {{
let decoded = decode_address_payload(address)?;
Ok(AddressHashPrefix {{
addr_type: decoded.addr_type,
prefix: address_payload_hash_prefix(&decoded.payload, nibbles)?,
}})
}}
/// Options for configuring the BRK client.
#[derive(Debug, Clone)]
pub struct BrkClientOptions {{
+1 -4
View File
@@ -25,12 +25,9 @@ owo-colors = { workspace = true }
tracing = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "1.1.2"
toml = "1.1.3"
vecdb = { workspace = true }
[[bin]]
name = "brk"
path = "src/main.rs"
[package.metadata.dist]
dist = true
+4
View File
@@ -30,6 +30,10 @@ Portable build (without native CPU optimizations):
cargo install --locked brk_cli
```
## Update
Updating is the same as installing: re-run the install command above. `cargo install` overwrites the existing binary in place. Indexed data is reused when the on-disk format is unchanged, otherwise it is reset and re-synced automatically on the next run.
## Run
```bash
+1
View File
@@ -13,6 +13,7 @@ exclude = ["examples/"]
[dependencies]
brk_cohort = { workspace = true }
brk_types = { workspace = true }
rapidhash = { workspace = true }
ureq = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
+1 -1
View File
@@ -17,7 +17,7 @@ fn main() -> brk_client::Result<()> {
// day1() returns DateMetricEndpointBuilder, so fetch() returns DateMetricData
let price_close = client
.series()
.prices
.price
.split
.close
.usd
File diff suppressed because it is too large Load Diff
+44
View File
@@ -0,0 +1,44 @@
use brk_client::{
BrkClient, OutputType, address_hash_prefix, address_payload_hash_prefix, decode_address_payload,
};
#[test]
fn address_payload_hash_prefix_vectors() {
let vectors = [
(vec![0x4e, 0x73], "58101afa51a1ecfd"),
((0_u8..20).collect::<Vec<_>>(), "c3327ecb8ae1ff23"),
((0_u8..32).collect::<Vec<_>>(), "c0186990f026b180"),
((0_u8..65).collect::<Vec<_>>(), "0d4b77027ae7d700"),
];
for (payload, expected) in vectors.iter() {
assert_eq!(address_payload_hash_prefix(payload, 16).unwrap(), *expected);
assert_eq!(
BrkClient::address_payload_hash_prefix(payload, 8).unwrap(),
&expected[..8]
);
}
}
#[test]
fn address_payload_hash_prefix_validation() {
assert!(address_payload_hash_prefix(&[], 16).is_err());
assert!(address_payload_hash_prefix(&[0; 66], 16).is_err());
assert!(address_payload_hash_prefix(&[1, 2], 0).is_err());
assert!(address_payload_hash_prefix(&[1, 2], 17).is_err());
}
#[test]
fn address_hash_prefix_uses_brk_address_parser() {
let address = "1BoatSLRHtKNngkdXEeobR76b53LETtpyT";
let decoded = decode_address_payload(address).unwrap();
assert_eq!(decoded.addr_type, OutputType::P2PKH);
assert_eq!(decoded.payload.len(), 20);
let hashed = address_hash_prefix(address, 8).unwrap();
assert_eq!(hashed.addr_type, OutputType::P2PKH);
assert_eq!(
hashed.prefix,
address_payload_hash_prefix(&decoded.payload, 8).unwrap()
);
}
+104
View File
@@ -0,0 +1,104 @@
use brk_traversable::Traversable;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use serde::Serialize;
use super::{CohortName, Filter};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EntryPrice {
Discount,
Premium,
}
impl EntryPrice {
#[inline]
pub const fn from_is_discount(is_discount: bool) -> Self {
if is_discount {
Self::Discount
} else {
Self::Premium
}
}
#[inline]
pub const fn is_discount(self) -> bool {
matches!(self, Self::Discount)
}
}
pub const ENTRY_FILTERS: ByEntry<Filter> = ByEntry {
discount: Filter::Entry(EntryPrice::Discount),
premium: Filter::Entry(EntryPrice::Premium),
};
pub const ENTRY_NAMES: ByEntry<CohortName> = ByEntry {
discount: CohortName::new("veteran", "Veteran", "Veteran Coins"),
premium: CohortName::new("rookie", "Rookie", "Rookie Coins"),
};
#[derive(Default, Clone, Traversable, Serialize)]
pub struct ByEntry<T> {
pub discount: T,
pub premium: T,
}
impl ByEntry<CohortName> {
pub const fn names() -> &'static Self {
&ENTRY_NAMES
}
}
impl<T> ByEntry<T> {
pub fn new<F>(mut create: F) -> Self
where
F: FnMut(Filter, &'static str) -> T,
{
let f = ENTRY_FILTERS;
let n = ENTRY_NAMES;
Self {
discount: create(f.discount, n.discount.id),
premium: create(f.premium, n.premium.id),
}
}
pub fn try_new<F, E>(mut create: F) -> Result<Self, E>
where
F: FnMut(Filter, &'static str) -> Result<T, E>,
{
let f = ENTRY_FILTERS;
let n = ENTRY_NAMES;
Ok(Self {
discount: create(f.discount, n.discount.id)?,
premium: create(f.premium, n.premium.id)?,
})
}
pub fn get(&self, entry: EntryPrice) -> &T {
match entry {
EntryPrice::Discount => &self.discount,
EntryPrice::Premium => &self.premium,
}
}
pub fn get_mut(&mut self, entry: EntryPrice) -> &mut T {
match entry {
EntryPrice::Discount => &mut self.discount,
EntryPrice::Premium => &mut self.premium,
}
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
[&self.discount, &self.premium].into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
[&mut self.discount, &mut self.premium].into_iter()
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
[&mut self.discount, &mut self.premium].into_par_iter()
}
}
+2 -1
View File
@@ -24,7 +24,7 @@ impl CohortContext {
/// Build full name for a filter, adding prefix only for Time/Amount filters.
///
/// Prefix rules:
/// - No prefix: `All`, `Term`, `Epoch`, `Class`, `Type`
/// - No prefix: `All`, `Term`, `Epoch`, `Class`, `Entry`, `Type`
/// - Context prefix: `Time`, `Amount`
pub fn full_name(&self, filter: &Filter, name: &str) -> String {
match filter {
@@ -32,6 +32,7 @@ impl CohortContext {
| Filter::Term(_)
| Filter::Epoch(_)
| Filter::Class(_)
| Filter::Entry(_)
| Filter::Type(_) => name.to_string(),
Filter::Time(_) | Filter::Amount(_) => self.prefixed(name),
}
+8 -3
View File
@@ -1,6 +1,6 @@
use brk_types::{Halving, OutputType, Sats, Year};
use super::{AmountFilter, CohortContext, Term, TimeFilter};
use super::{AmountFilter, CohortContext, EntryPrice, Term, TimeFilter};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Filter {
@@ -10,6 +10,7 @@ pub enum Filter {
Amount(AmountFilter),
Epoch(Halving),
Class(Year),
Entry(EntryPrice),
Type(OutputType),
}
@@ -68,7 +69,8 @@ impl Filter {
}
/// Whether to compute extended metrics (realized cap ratios, profit/loss ratios, percentiles)
/// For UTXO context: true only for age range cohorts (Range) and aggregate cohorts (All, Term)
/// For UTXO context: true for age range cohorts (Range), aggregate cohorts (All, Term),
/// and immutable entry valuation cohorts.
/// For address context: always false
pub fn is_extended(&self, context: CohortContext) -> bool {
match context {
@@ -76,7 +78,10 @@ impl Filter {
CohortContext::Utxo => {
matches!(
self,
Filter::All | Filter::Term(_) | Filter::Time(TimeFilter::Range(_))
Filter::All
| Filter::Term(_)
| Filter::Time(TimeFilter::Range(_))
| Filter::Entry(_)
)
}
}
+2
View File
@@ -7,6 +7,7 @@ mod amount_range;
mod by_addr_type;
mod by_any_addr;
mod by_epoch;
mod by_entry;
mod by_term;
mod by_type;
mod class;
@@ -36,6 +37,7 @@ pub use amount_range::*;
pub use by_addr_type::*;
pub use by_any_addr::*;
pub use by_epoch::*;
pub use by_entry::*;
pub use by_term::*;
pub use by_type::*;
pub use class::*;
+10 -2
View File
@@ -2,8 +2,8 @@ use brk_traversable::Traversable;
use rayon::prelude::*;
use crate::{
AgeRange, AmountRange, ByEpoch, ByTerm, Class, Filter, OverAge, OverAmount, SpendableType,
UnderAge, UnderAmount,
AgeRange, AmountRange, ByEntry, ByEpoch, ByTerm, Class, Filter, OverAge, OverAmount,
SpendableType, UnderAge, UnderAmount,
};
#[derive(Default, Clone, Traversable)]
@@ -12,6 +12,7 @@ pub struct UTXOGroups<T> {
pub age_range: AgeRange<T>,
pub epoch: ByEpoch<T>,
pub class: Class<T>,
pub entry: ByEntry<T>,
pub over_age: OverAge<T>,
pub over_amount: OverAmount<T>,
pub amount_range: AmountRange<T>,
@@ -31,6 +32,7 @@ impl<T> UTXOGroups<T> {
age_range: AgeRange::new(&mut create),
epoch: ByEpoch::new(&mut create),
class: Class::new(&mut create),
entry: ByEntry::new(&mut create),
over_age: OverAge::new(&mut create),
over_amount: OverAmount::new(&mut create),
amount_range: AmountRange::new(&mut create),
@@ -51,6 +53,7 @@ impl<T> UTXOGroups<T> {
.chain(self.age_range.iter())
.chain(self.epoch.iter())
.chain(self.class.iter())
.chain(self.entry.iter())
.chain(self.amount_range.iter())
.chain(self.under_amount.iter())
.chain(self.type_.iter())
@@ -66,6 +69,7 @@ impl<T> UTXOGroups<T> {
.chain(self.age_range.iter_mut())
.chain(self.epoch.iter_mut())
.chain(self.class.iter_mut())
.chain(self.entry.iter_mut())
.chain(self.amount_range.iter_mut())
.chain(self.under_amount.iter_mut())
.chain(self.type_.iter_mut())
@@ -84,6 +88,7 @@ impl<T> UTXOGroups<T> {
.chain(self.age_range.par_iter_mut())
.chain(self.epoch.par_iter_mut())
.chain(self.class.par_iter_mut())
.chain(self.entry.par_iter_mut())
.chain(self.amount_range.par_iter_mut())
.chain(self.under_amount.par_iter_mut())
.chain(self.type_.par_iter_mut())
@@ -94,6 +99,7 @@ impl<T> UTXOGroups<T> {
.iter()
.chain(self.epoch.iter())
.chain(self.class.iter())
.chain(self.entry.iter())
.chain(self.amount_range.iter())
.chain(self.type_.iter())
}
@@ -103,6 +109,7 @@ impl<T> UTXOGroups<T> {
.iter_mut()
.chain(self.epoch.iter_mut())
.chain(self.class.iter_mut())
.chain(self.entry.iter_mut())
.chain(self.amount_range.iter_mut())
.chain(self.type_.iter_mut())
}
@@ -115,6 +122,7 @@ impl<T> UTXOGroups<T> {
.par_iter_mut()
.chain(self.epoch.par_iter_mut())
.chain(self.class.par_iter_mut())
.chain(self.entry.par_iter_mut())
.chain(self.amount_range.par_iter_mut())
.chain(self.type_.par_iter_mut())
}
+2 -2
View File
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, distribution, mining, prices, supply};
use crate::{blocks, distribution, mining, price, supply};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
mining: &mining::Vecs,
supply_vecs: &supply::Vecs,
@@ -5,14 +5,14 @@ use vecdb::Exit;
use super::super::{activity, cap, supply};
use super::Vecs;
use crate::{distribution, prices};
use crate::{distribution, price};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
supply: &supply::Vecs,
@@ -4,14 +4,14 @@ use brk_types::StoredF64;
use vecdb::Exit;
use super::{super::value, Vecs};
use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, prices};
use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
value: &value::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -4,13 +4,13 @@ use vecdb::Exit;
use super::super::activity;
use super::Vecs;
use crate::{distribution, prices};
use crate::{distribution, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
exit: &Exit,
@@ -5,13 +5,13 @@ use vecdb::Exit;
use super::super::activity;
use super::Vecs;
use crate::{distribution, prices};
use crate::{distribution, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
exit: &Exit,
@@ -45,7 +45,7 @@ use super::{
count::AddrCountFundedTotalVecs,
supply::{AddrSupplyShareVecs, AddrSupplyVecs},
};
use crate::{indexes, prices};
use crate::{indexes, price};
mod state;
@@ -104,7 +104,7 @@ impl ExposedAddrVecs {
pub(crate) fn compute_rest(
&mut self,
starting_lengths: &Lengths,
prices: &prices::Vecs,
prices: &price::Vecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
@@ -35,7 +35,7 @@ use super::{
use crate::{
indexes, inputs,
internal::{WindowStartVec, Windows},
outputs, prices,
outputs, price,
};
mod state;
@@ -112,7 +112,7 @@ impl ReusedAddrVecs {
starting_lengths: &Lengths,
outputs_by_type: &outputs::ByTypeVecs,
inputs_by_type: &inputs::ByTypeVecs,
prices: &prices::Vecs,
prices: &price::Vecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
@@ -13,7 +13,7 @@ use crate::{
distribution::DynCohortVecs,
indexes,
internal::{WindowStartVec, Windows},
prices,
price,
};
use super::{super::traits::CohortVecs, vecs::AddrCohortVecs};
@@ -95,7 +95,7 @@ impl AddrCohorts {
/// First phase of post-processing: compute index transforms.
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -108,7 +108,7 @@ impl AddrCohorts {
/// Second phase of post-processing: compute relative metrics.
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -12,7 +12,7 @@ use crate::{
distribution::state::{AddrCohortState, MinimalRealizedState},
indexes,
internal::{PerBlockWithDeltas, WindowStartVec, Windows},
prices,
price,
};
use crate::distribution::metrics::{ImportConfig, MinimalCohortMetrics};
@@ -174,7 +174,7 @@ impl DynCohortVecs for AddrCohortVecs {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -229,7 +229,7 @@ impl CohortVecs for AddrCohortVecs {
fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -3,7 +3,7 @@ use brk_indexer::Lengths;
use brk_types::{Cents, Height, Sats, StoredU64, Version};
use vecdb::{Exit, ReadableVec};
use crate::prices;
use crate::price;
/// Dynamic dispatch trait for cohort vectors.
///
@@ -31,7 +31,7 @@ pub trait DynCohortVecs: Send + Sync {
/// First phase of post-processing computations.
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()>;
@@ -61,7 +61,7 @@ pub trait CohortVecs: DynCohortVecs {
/// Second phase of post-processing computations.
fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -30,18 +30,34 @@ const TREE_SIZE: usize = TIER0_COUNT + TIER1_COUNT + OVERFLOW; // 190,001
pub(super) struct CostBasisNode {
all_sats: i64,
sth_sats: i64,
discount_sats: i64,
all_usd: i128,
sth_usd: i128,
discount_usd: i128,
}
impl CostBasisNode {
#[inline(always)]
fn new(sats: i64, usd: i128, is_sth: bool) -> Self {
fn new_supply(sats: i64, usd: i128, is_sth: bool) -> Self {
Self {
all_sats: sats,
sth_sats: if is_sth { sats } else { 0 },
discount_sats: 0,
all_usd: usd,
sth_usd: if is_sth { usd } else { 0 },
discount_usd: 0,
}
}
#[inline(always)]
fn new_discount(sats: i64, usd: i128) -> Self {
Self {
all_sats: 0,
sth_sats: 0,
discount_sats: sats,
all_usd: 0,
sth_usd: 0,
discount_usd: usd,
}
}
}
@@ -51,8 +67,10 @@ impl FenwickNode for CostBasisNode {
fn add_assign(&mut self, other: &Self) {
self.all_sats += other.all_sats;
self.sth_sats += other.sth_sats;
self.discount_sats += other.discount_sats;
self.all_usd += other.all_usd;
self.sth_usd += other.sth_usd;
self.discount_usd += other.discount_usd;
}
}
@@ -151,16 +169,34 @@ impl CostBasisFenwick {
}
let bucket = price_to_bucket(price);
let delta =
CostBasisNode::new(net_sats, price.as_u128() as i128 * net_sats as i128, is_sth);
CostBasisNode::new_supply(net_sats, price.as_u128() as i128 * net_sats as i128, is_sth);
self.tree.add(bucket, &delta);
self.totals.add_assign(&delta);
}
/// Bulk-initialize from BTreeMaps (one per age-range cohort).
/// Call after state import when all pending maps have been drained.
pub(super) fn bulk_init<'a>(
/// Apply a net delta from the discount-entry cohort.
///
/// Supply totals are maintained from the age-range cohorts; this updates
/// only the discount-entry partition so premium can be derived as all - discount.
pub(super) fn apply_discount_delta(&mut self, price: CentsCompact, pending: &PendingDelta) {
let net_sats = u64::from(pending.inc) as i64 - u64::from(pending.dec) as i64;
if net_sats == 0 {
return;
}
let bucket = price_to_bucket(price);
let delta =
CostBasisNode::new_discount(net_sats, price.as_u128() as i128 * net_sats as i128);
self.tree.add(bucket, &delta);
self.totals.add_assign(&delta);
}
/// Bulk-initialize from age-range maps plus the discount-entry map.
/// Age-range maps maintain all/STH/LTH totals; the discount-entry map
/// maintains only the discount partition used to derive premium.
pub(super) fn bulk_init_with_discount<'a>(
&mut self,
maps: impl Iterator<Item = (&'a std::collections::BTreeMap<CentsCompact, Sats>, bool)>,
discount_maps: impl Iterator<Item = &'a std::collections::BTreeMap<CentsCompact, Sats>>,
) {
self.tree.reset();
self.totals = CostBasisNode::default();
@@ -169,7 +205,18 @@ impl CostBasisFenwick {
for (&price, &sats) in map.iter() {
let bucket = price_to_bucket(price);
let s = u64::from(sats) as i64;
let node = CostBasisNode::new(s, price.as_u128() as i128 * s as i128, is_sth);
let node =
CostBasisNode::new_supply(s, price.as_u128() as i128 * s as i128, is_sth);
self.tree.add_raw(bucket, &node);
self.totals.add_assign(&node);
}
}
for map in discount_maps {
for (&price, &sats) in map.iter() {
let bucket = price_to_bucket(price);
let s = u64::from(sats) as i64;
let node = CostBasisNode::new_discount(s, price.as_u128() as i128 * s as i128);
self.tree.add_raw(bucket, &node);
self.totals.add_assign(&node);
}
@@ -212,6 +259,26 @@ impl CostBasisFenwick {
)
}
/// Compute percentile prices for discount-entry cohort.
pub(super) fn percentiles_discount_entry(&self) -> PercentileResult {
self.compute_percentiles(
self.totals.discount_sats,
self.totals.discount_usd,
|n| n.discount_sats,
|n| n.discount_usd,
)
}
/// Compute percentile prices for premium-entry cohort (all - discount).
pub(super) fn percentiles_premium_entry(&self) -> PercentileResult {
self.compute_percentiles(
self.totals.all_sats - self.totals.discount_sats,
self.totals.all_usd - self.totals.discount_usd,
|n| n.all_sats - n.discount_sats,
|n| n.all_usd - n.discount_usd,
)
}
fn compute_percentiles(
&self,
total_sats: i64,
@@ -271,6 +338,37 @@ impl CostBasisFenwick {
return (0, 0, 0);
}
let range = self.density_range(spot_price);
let all_range = range.all_sats.max(0);
let sth_range = range.sth_sats.max(0);
let lth_range = all_range - sth_range;
let lth_total = self.totals.all_sats - self.totals.sth_sats;
(
Self::to_bps(all_range, self.totals.all_sats),
Self::to_bps(sth_range, self.totals.sth_sats),
Self::to_bps(lth_range, lth_total),
)
}
/// Compute supply density for entry cohorts: (discount_bps, premium_bps).
pub(super) fn entry_density(&self, spot_price: Cents) -> (u16, u16) {
if self.totals.all_sats <= 0 {
return (0, 0);
}
let range = self.density_range(spot_price);
let discount_range = range.discount_sats.max(0);
let premium_range = range.all_sats.max(0) - discount_range;
let premium_total = self.totals.all_sats - self.totals.discount_sats;
(
Self::to_bps(discount_range, self.totals.discount_sats),
Self::to_bps(premium_range, premium_total),
)
}
fn density_range(&self, spot_price: Cents) -> CostBasisNode {
let spot_f64 = u64::from(spot_price) as f64;
let low = Cents::from((spot_f64 * 0.95) as u64);
let high = Cents::from((spot_f64 * 1.05) as u64);
@@ -285,24 +383,23 @@ impl CostBasisFenwick {
CostBasisNode::default()
};
let all_range = (cum_high.all_sats - cum_low.all_sats).max(0);
let sth_range = (cum_high.sth_sats - cum_low.sth_sats).max(0);
let lth_range = all_range - sth_range;
CostBasisNode {
all_sats: cum_high.all_sats - cum_low.all_sats,
sth_sats: cum_high.sth_sats - cum_low.sth_sats,
discount_sats: cum_high.discount_sats - cum_low.discount_sats,
all_usd: cum_high.all_usd - cum_low.all_usd,
sth_usd: cum_high.sth_usd - cum_low.sth_usd,
discount_usd: cum_high.discount_usd - cum_low.discount_usd,
}
}
let to_bps = |range: i64, total: i64| -> u16 {
if total <= 0 {
0
} else {
(range as f64 / total as f64 * 10000.0).round() as u16
}
};
let lth_total = self.totals.all_sats - self.totals.sth_sats;
(
to_bps(all_range, self.totals.all_sats),
to_bps(sth_range, self.totals.sth_sats),
to_bps(lth_range, lth_total),
)
#[inline(always)]
fn to_bps(range: i64, total: i64) -> u16 {
if total <= 0 {
0
} else {
(range as f64 / total as f64 * 10000.0).round() as u16
}
}
// -----------------------------------------------------------------------
@@ -1,8 +1,8 @@
use std::path::Path;
use brk_cohort::{
AgeRange, AmountRange, ByEpoch, Class, CohortContext, Filter, Filtered, OverAge, OverAmount,
SpendableType, Term, UnderAge, UnderAmount,
AgeRange, AmountRange, ByEntry, ByEpoch, Class, CohortContext, Filter, Filtered, OverAge,
OverAmount, SpendableType, Term, UnderAge, UnderAmount,
};
use brk_error::Result;
use brk_indexer::Lengths;
@@ -27,7 +27,7 @@ use crate::{
},
indexes,
internal::{ValuePerBlockCumulativeRolling, WindowStartVec, Windows},
prices,
price,
};
use super::{fenwick::CostBasisFenwick, vecs::UTXOCohortVecs};
@@ -45,6 +45,7 @@ pub struct UTXOCohorts<M: StorageMode = Rw> {
pub over_age: OverAge<UTXOCohortVecs<CoreCohortMetrics<M>>>,
pub epoch: ByEpoch<UTXOCohortVecs<CoreCohortMetrics<M>>>,
pub class: Class<UTXOCohortVecs<CoreCohortMetrics<M>>>,
pub entry: ByEntry<UTXOCohortVecs<ExtendedCohortMetrics<M>>>,
pub over_amount: OverAmount<UTXOCohortVecs<MinimalCohortMetrics<M>>>,
pub amount_range: AmountRange<UTXOCohortVecs<MinimalCohortMetrics<M>>>,
pub under_amount: UnderAmount<UTXOCohortVecs<MinimalCohortMetrics<M>>>,
@@ -67,8 +68,10 @@ pub(crate) struct UTXOCohortsTransientState {
}
impl UTXOCohorts<Rw> {
/// ~71 separate cohorts (21 age + 5 epoch + 18 class + 15 amount + 12 type)
const SEPARATE_COHORT_CAPACITY: usize = 80;
/// Separate cohorts currently total 72:
/// 21 age + 5 epoch + 18 class + 2 entry + 15 amount + 11 spendable type.
/// Keep small headroom because this is only Vec allocation capacity.
const SEPARATE_COHORT_CAPACITY: usize = 82;
/// Import all UTXO cohorts from database.
pub(crate) fn forced_import(
@@ -136,6 +139,26 @@ impl UTXOCohorts<Rw> {
let epoch = ByEpoch::try_new(&core_separate)?;
let class = Class::try_new(&core_separate)?;
let extended_separate =
|f: Filter, name: &'static str| -> Result<UTXOCohortVecs<ExtendedCohortMetrics>> {
let full_name = CohortContext::Utxo.full_name(&f, name);
let cfg = ImportConfig {
db,
filter: &f,
full_name: &full_name,
version: v,
indexes,
cached_starts,
};
let state = Some(Box::new(UTXOCohortState::new(states_path, &full_name)));
Ok(UTXOCohortVecs::new(
state,
ExtendedCohortMetrics::forced_import(&cfg)?,
))
};
let entry = ByEntry::try_new(&extended_separate)?;
// Helper for separate cohorts with MinimalCohortMetrics + MinimalRealizedState
let minimal_separate =
|f: Filter, name: &'static str| -> Result<UTXOCohortVecs<MinimalCohortMetrics>> {
@@ -281,6 +304,7 @@ impl UTXOCohorts<Rw> {
lth,
epoch,
class,
entry,
type_,
under_age,
over_age,
@@ -309,6 +333,7 @@ impl UTXOCohorts<Rw> {
sth,
caches,
age_range,
entry,
..
} = self;
caches
@@ -327,7 +352,15 @@ impl UTXOCohorts<Rw> {
Some((map, caches.fenwick.is_sth_at(i)))
})
.collect();
caches.fenwick.bulk_init(maps.into_iter());
let discount_maps = entry
.discount
.state
.as_ref()
.map(|state| state.cost_basis_map())
.into_iter();
caches
.fenwick
.bulk_init_with_discount(maps.into_iter(), discount_maps);
}
/// Apply pending deltas from all age-range cohorts to the Fenwick tree.
@@ -338,7 +371,10 @@ impl UTXOCohorts<Rw> {
}
// Destructure to get separate borrows on caches and age_range
let Self {
caches, age_range, ..
caches,
age_range,
entry,
..
} = self;
for (i, sub) in age_range.iter().enumerate() {
if let Some(state) = sub.state.as_ref() {
@@ -348,6 +384,11 @@ impl UTXOCohorts<Rw> {
});
}
}
if let Some(state) = entry.discount.state.as_ref() {
state.for_each_cost_basis_pending(|&price, delta| {
caches.fenwick.apply_discount_delta(price, delta);
});
}
}
/// Push maturation sats to the matured vecs for the given height.
@@ -365,6 +406,7 @@ impl UTXOCohorts<Rw> {
age_range,
epoch,
class,
entry,
amount_range,
type_,
..
@@ -374,6 +416,7 @@ impl UTXOCohorts<Rw> {
.map(|x| x as &mut dyn DynCohortVecs)
.chain(epoch.par_iter_mut().map(|x| x as &mut dyn DynCohortVecs))
.chain(class.par_iter_mut().map(|x| x as &mut dyn DynCohortVecs))
.chain(entry.par_iter_mut().map(|x| x as &mut dyn DynCohortVecs))
.chain(
amount_range
.par_iter_mut()
@@ -389,6 +432,7 @@ impl UTXOCohorts<Rw> {
age_range,
epoch,
class,
entry,
amount_range,
type_,
..
@@ -398,6 +442,7 @@ impl UTXOCohorts<Rw> {
.map(|x| x as &mut dyn DynCohortVecs)
.chain(epoch.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
.chain(class.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
.chain(entry.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
.chain(amount_range.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
.chain(type_.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
}
@@ -409,6 +454,7 @@ impl UTXOCohorts<Rw> {
.map(|x| x as &dyn DynCohortVecs)
.chain(self.epoch.iter().map(|x| x as &dyn DynCohortVecs))
.chain(self.class.iter().map(|x| x as &dyn DynCohortVecs))
.chain(self.entry.iter().map(|x| x as &dyn DynCohortVecs))
.chain(self.amount_range.iter().map(|x| x as &dyn DynCohortVecs))
.chain(self.type_.iter().map(|x| x as &dyn DynCohortVecs))
}
@@ -483,7 +529,7 @@ impl UTXOCohorts<Rw> {
/// First phase of post-processing: compute index transforms.
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -516,6 +562,7 @@ impl UTXOCohorts<Rw> {
);
all.extend(self.epoch.iter_mut().map(|x| x as &mut dyn DynCohortVecs));
all.extend(self.class.iter_mut().map(|x| x as &mut dyn DynCohortVecs));
all.extend(self.entry.iter_mut().map(|x| x as &mut dyn DynCohortVecs));
all.extend(
self.amount_range
.iter_mut()
@@ -546,7 +593,7 @@ impl UTXOCohorts<Rw> {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
exit: &Exit,
@@ -604,6 +651,7 @@ impl UTXOCohorts<Rw> {
under_amount,
epoch,
class,
entry,
type_,
..
} = self;
@@ -676,6 +724,19 @@ impl UTXOCohorts<Rw> {
.compute_rest_part2(prices, starting_lengths, ss, au, exit)
})
}),
Box::new(|| {
entry.par_iter_mut().try_for_each(|v| {
v.metrics.compute_rest_part2(
blocks,
prices,
starting_lengths,
height_to_market_cap,
ss,
au,
exit,
)
})
}),
Box::new(|| {
amount_range.par_iter_mut().try_for_each(|v| {
v.metrics
@@ -730,6 +791,9 @@ impl UTXOCohorts<Rw> {
for v in self.class.iter_mut() {
vecs.extend(v.metrics.collect_all_vecs_mut());
}
for v in self.entry.iter_mut() {
vecs.extend(v.metrics.collect_all_vecs_mut());
}
for v in self.amount_range.iter_mut() {
vecs.extend(v.metrics.collect_all_vecs_mut());
}
@@ -813,7 +877,7 @@ impl UTXOCohorts<Rw> {
/// Aggregate RealizedFull fields from age_range states and push to all/sth/lth.
/// Called during the block loop after separate cohorts' push_state but before reset.
pub(crate) fn push_overlapping(&mut self, height_price: Cents) {
pub(crate) fn push_overlapping(&mut self, height_price: Cents) -> Cents {
let Self {
all,
sth,
@@ -852,7 +916,7 @@ impl UTXOCohorts<Rw> {
}
}
all.metrics.realized.push_accum(&all_acc);
let all_capitalized_price = all.metrics.realized.push_accum(&all_acc);
sth.metrics.realized.push_accum(&sth_acc);
lth.metrics.realized.push_accum(&lth_acc);
@@ -880,6 +944,8 @@ impl UTXOCohorts<Rw> {
.unrealized
.capitalized_cap_in_loss_raw
.push(CentsSquaredSats::new(lth_ccap.1));
all_capitalized_price
}
}
@@ -50,6 +50,22 @@ impl UTXOCohorts {
let lth = self.caches.fenwick.percentiles_lth();
push_cost_basis(&lth, lth_d, &mut self.lth.metrics.cost_basis);
let (discount_d, premium_d) = self.caches.fenwick.entry_density(spot_price);
let discount = self.caches.fenwick.percentiles_discount_entry();
push_cost_basis(
&discount,
discount_d,
&mut self.entry.discount.metrics.cost_basis,
);
let premium = self.caches.fenwick.percentiles_premium_entry();
push_cost_basis(
&premium,
premium_d,
&mut self.entry.premium.metrics.cost_basis,
);
let prof = self.caches.fenwick.profitability(spot_price);
push_profitability(&prof, &mut self.profitability);
}
@@ -1,3 +1,4 @@
use brk_cohort::EntryPrice;
use brk_types::{Cents, CostBasisSnapshot, Height, Timestamp};
use vecdb::Rw;
@@ -12,6 +13,7 @@ impl UTXOCohorts<Rw> {
/// - The "under_1h" age cohort (all new UTXOs start at 0 hours old)
/// - The appropriate epoch cohort based on block height
/// - The appropriate class cohort based on block timestamp
/// - The immutable entry valuation cohort based on creation price versus anchor
/// - The appropriate output type cohort (P2PKH, P2SH, etc.)
/// - The appropriate amount range cohort based on value
pub(crate) fn receive(
@@ -20,13 +22,14 @@ impl UTXOCohorts<Rw> {
height: Height,
timestamp: Timestamp,
price: Cents,
entry: EntryPrice,
) {
let supply_state = received.spendable_supply;
// Pre-compute snapshot once for the 3 cohorts sharing the same supply_state
// Pre-compute snapshot once for cohorts sharing the block-level supply_state
let snapshot = CostBasisSnapshot::from_utxo(price, &supply_state);
// New UTXOs go into under_1h, current epoch, and current class
// New UTXOs go into under_1h plus immutable creation cohorts
self.age_range
.under_1h
.state
@@ -45,6 +48,12 @@ impl UTXOCohorts<Rw> {
.unwrap()
.receive_utxo_snapshot(&supply_state, &snapshot);
}
self.entry
.get_mut(entry)
.state
.as_mut()
.unwrap()
.receive_utxo_snapshot(&supply_state, &snapshot);
// Update output type cohorts (skip types with no outputs this block)
self.type_.iter_typed_mut().for_each(|(output_type, vecs)| {
@@ -49,7 +49,7 @@ impl UTXOCohorts<Rw> {
// This is the max price between receive and send heights
let peak_price = price_range_max.max_between(receive_height, send_height);
// Pre-compute once for age_range, epoch, year (all share sent.spendable_supply)
// Pre-compute once for cohorts sharing the sent supply.
if let Some(pre) = SendPrecomputed::new(
&sent.spendable_supply,
current_price,
@@ -75,6 +75,12 @@ impl UTXOCohorts<Rw> {
.unwrap()
.send_utxo_precomputed(&sent.spendable_supply, &pre);
}
self.entry
.get_mut(block_state.entry)
.state
.as_mut()
.unwrap()
.send_utxo_precomputed(&sent.spendable_supply, &pre);
} else if sent.spendable_supply.utxo_count > 0 {
// Zero-value UTXOs: just subtract supply
self.age_range.get_mut(age).state.as_mut().unwrap().supply -=
@@ -85,6 +91,12 @@ impl UTXOCohorts<Rw> {
if let Some(v) = self.class.mut_vec_from_timestamp(block_state.timestamp) {
v.state.as_mut().unwrap().supply -= &sent.spendable_supply;
}
self.entry
.get_mut(block_state.entry)
.state
.as_mut()
.unwrap()
.supply -= &sent.spendable_supply;
}
// Update output type cohorts (skip zero-supply entries)
@@ -6,7 +6,7 @@ use vecdb::{Exit, ReadableVec};
use crate::{
distribution::{cohorts::traits::DynCohortVecs, metrics::CoreCohortMetrics},
prices,
price,
};
use super::UTXOCohortVecs;
@@ -56,7 +56,7 @@ impl DynCohortVecs for UTXOCohortVecs<CoreCohortMetrics> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -6,7 +6,7 @@ use vecdb::{Exit, ReadableVec};
use crate::{
distribution::{cohorts::traits::DynCohortVecs, metrics::MinimalCohortMetrics},
prices,
price,
};
use super::UTXOCohortVecs;
@@ -49,7 +49,7 @@ impl DynCohortVecs for UTXOCohortVecs<MinimalCohortMetrics> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -55,7 +55,7 @@ use crate::{
metrics::{CohortMetricsBase, CohortMetricsState},
state::UTXOCohortState,
},
prices,
price,
};
#[derive(Traversable)]
@@ -186,7 +186,7 @@ impl<M: CohortMetricsBase + Traversable> DynCohortVecs for UTXOCohortVecs<M> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -5,7 +5,7 @@ use brk_types::{Cents, Height, Version};
use vecdb::{Exit, ReadableVec};
use crate::{
distribution::cohorts::traits::DynCohortVecs, distribution::metrics::TypeCohortMetrics, prices,
distribution::cohorts::traits::DynCohortVecs, distribution::metrics::TypeCohortMetrics, price,
};
use super::UTXOCohortVecs;
@@ -55,7 +55,7 @@ impl DynCohortVecs for UTXOCohortVecs<TypeCohortMetrics> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -1,4 +1,4 @@
use brk_cohort::ByAddrType;
use brk_cohort::{ByAddrType, EntryPrice};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{
@@ -46,6 +46,7 @@ pub(crate) fn process_blocks(
last_height: Height,
chain_state: &mut Vec<BlockState>,
tx_index_to_height: &mut RangeMap<TxIndex, Height>,
mut entry_anchor: Cents,
cached_prices: &[Cents],
cached_timestamps: &[Timestamp],
cached_price_range_max: &PriceRangeMax,
@@ -370,9 +371,14 @@ pub(crate) fn process_blocks(
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
}
let entry = EntryPrice::from_is_discount(
entry_anchor == Cents::ZERO || block_price <= entry_anchor,
);
// Push current block state before processing cohort updates
chain_state.push(BlockState {
supply: transacted.spendable_supply,
entry,
price: block_price,
timestamp,
});
@@ -411,7 +417,7 @@ pub(crate) fn process_blocks(
|| {
// UTXO cohorts receive/send
vecs.utxo_cohorts
.receive(transacted, height, timestamp, block_price);
.receive(transacted, height, timestamp, block_price, entry);
if let Some(min_h) =
vecs.utxo_cohorts
.send(height_to_sent, chain_state, ctx.price_range_max)
@@ -460,7 +466,7 @@ pub(crate) fn process_blocks(
let is_last_of_day = is_last_of_day[offset];
let date_opt = is_last_of_day.then(|| Date::from(timestamp));
push_cohort_states(
entry_anchor = push_cohort_states(
&mut vecs.utxo_cohorts,
&mut vecs.addr_cohorts,
height,
@@ -527,7 +533,7 @@ fn push_cohort_states(
addr_cohorts: &mut AddrCohorts,
height: Height,
height_price: Cents,
) {
) -> Cents {
// Phase 1: push + unrealized (no reset yet, states still needed for aggregation)
rayon::join(
|| {
@@ -545,7 +551,7 @@ fn push_cohort_states(
);
// Phase 2: aggregate age_range states → push to overlapping cohorts
utxo_cohorts.push_overlapping(height_price);
let all_capitalized_price = utxo_cohorts.push_overlapping(height_price);
// Phase 3: reset per-block values
utxo_cohorts
@@ -554,4 +560,6 @@ fn push_cohort_states(
addr_cohorts
.iter_separate_mut()
.for_each(|v| v.reset_single_iteration_values());
all_capitalized_price
}
@@ -24,51 +24,60 @@ pub struct RecoveredState {
/// Returns Height::ZERO if any validation fails (triggers fresh start).
pub(crate) fn recover_state(
height: Height,
chain_state_rollback: vecdb::Result<Stamp>,
chain_state_rollback: Option<vecdb::Result<Stamp>>,
any_addr_indexes: &mut AnyAddrIndexesVecs,
addrs_data: &mut AddrsDataVecs,
utxo_cohorts: &mut UTXOCohorts,
addr_cohorts: &mut AddrCohorts,
) -> Result<RecoveredState> {
let stamp = Stamp::from(height);
// `None`: clean resume, already at the checkpoint, nothing to undo.
// `Some`: reorg, undo state past the resume point.
let consistent_height = match chain_state_rollback {
None => height,
Some(chain_state_rollback) => {
let stamp = Stamp::from(height);
// Rollback address state vectors
let addr_indexes_rollback = any_addr_indexes.rollback_before(stamp);
let addr_data_rollback = addrs_data.rollback_before(stamp);
// Rollback address state vectors
let addr_indexes_rollback = any_addr_indexes.rollback_before(stamp);
let addr_data_rollback = addrs_data.rollback_before(stamp);
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
addr_indexes_rollback,
addr_data_rollback,
);
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
addr_indexes_rollback,
addr_data_rollback,
);
// If rollbacks are inconsistent, start fresh
if consistent_height.is_zero() {
warn!("Rollback consistency check failed: inconsistent heights");
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// If rollbacks are inconsistent, start fresh
if consistent_height.is_zero() {
warn!("Rollback consistency check failed: inconsistent heights");
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Rollback can land at an earlier height (multi-block change file), which is fine.
// But if it lands AHEAD of target, that means rollback failed (missing change files).
if consistent_height > height {
warn!(
"Rollback failed: still at {} but target was {}, falling back to fresh start",
consistent_height, height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Rollback can land at an earlier height (multi-block change file), which is fine.
// But if it lands AHEAD of target, that means rollback failed (missing change files).
if consistent_height > height {
warn!(
"Rollback failed: still at {} but target was {}, falling back to fresh start",
consistent_height, height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
if consistent_height != height {
debug!(
"Rollback landed at {} instead of {}, will resume from there",
consistent_height, height
);
}
if consistent_height != height {
debug!(
"Rollback landed at {} instead of {}, will resume from there",
consistent_height, height
);
}
consistent_height
}
};
// Import UTXO cohort states - all must succeed
debug!(
@@ -11,7 +11,7 @@ use crate::{
state::{CohortState, CostBasisOps, RealizedOps},
},
internal::{PerBlockCumulativeRolling, ValuePerBlockCumulativeRolling},
prices,
price,
};
use super::ActivityMinimal;
@@ -98,7 +98,7 @@ impl ActivityCore {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -12,7 +12,7 @@ use crate::{
metrics::ImportConfig,
state::{CohortState, CostBasisOps, RealizedOps},
},
prices,
price,
};
use super::ActivityCore;
@@ -89,7 +89,7 @@ impl ActivityFull {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -10,7 +10,7 @@ use crate::{
state::{CohortState, CostBasisOps, RealizedOps},
},
internal::ValuePerBlockCumulativeRolling,
prices,
price,
};
#[derive(Traversable)]
@@ -63,7 +63,7 @@ impl ActivityMinimal {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -13,7 +13,7 @@ use vecdb::Exit;
use crate::{
distribution::state::{CohortState, CostBasisOps, RealizedOps},
prices,
price,
};
pub trait ActivityLike: Send + Sync {
@@ -30,7 +30,7 @@ pub trait ActivityLike: Send + Sync {
) -> Result<()>;
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()>;
@@ -62,7 +62,7 @@ impl ActivityLike for ActivityCore {
}
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -96,7 +96,7 @@ impl ActivityLike for ActivityFull {
}
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -11,7 +11,7 @@ use crate::{
ActivityFull, AdjustedSopr, CohortMetricsBase, CostBasis, ImportConfig, OutputsBase,
RealizedFull, RelativeForAll, SupplyCore, UnrealizedFull,
},
prices,
price,
};
/// All-cohort metrics: extended realized + adjusted (as composable add-on),
@@ -100,7 +100,7 @@ impl AllCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
under_1h_value_created: &impl ReadableVec<Height, Cents>,
@@ -10,7 +10,7 @@ use crate::{
ActivityCore, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore, SupplyCore,
UnrealizedCore,
},
prices,
price,
};
/// Basic cohort metrics: no extensions, used by age_range cohorts.
@@ -61,7 +61,7 @@ impl BasicCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -10,7 +10,7 @@ use crate::{
ActivityCore, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore, SupplyCore,
UnrealizedCore,
},
prices,
price,
};
#[derive(Traversable)]
@@ -102,7 +102,7 @@ impl CoreCohortMetrics {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -122,7 +122,7 @@ impl CoreCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -12,7 +12,7 @@ use crate::{
ActivityFull, CohortMetricsBase, CostBasis, ImportConfig, OutputsBase, RealizedFull,
RelativeWithExtended, SupplyCore, UnrealizedFull,
},
prices,
price,
};
/// Cohort metrics with extended realized + extended cost basis (no adjusted).
@@ -90,7 +90,7 @@ impl ExtendedCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
all_supply_sats: &impl ReadableVec<Height, Sats>,
@@ -10,7 +10,7 @@ use crate::{
distribution::metrics::{
ActivityFull, AdjustedSopr, CohortMetricsBase, ImportConfig, RealizedFull, UnrealizedFull,
},
prices,
price,
};
use super::ExtendedCohortMetrics;
@@ -62,7 +62,7 @@ impl ExtendedAdjustedCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
under_1h_value_created: &impl ReadableVec<Height, Cents>,
@@ -9,7 +9,7 @@ use crate::{
distribution::metrics::{
ActivityMinimal, ImportConfig, OutputsBase, RealizedMinimal, SupplyBase, UnrealizedMinimal,
},
prices,
price,
};
/// MinimalCohortMetrics: supply, outputs, realized cap/price/mvrv/profit/loss + value_created/destroyed.
@@ -97,7 +97,7 @@ impl MinimalCohortMetrics {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -111,7 +111,7 @@ impl MinimalCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -9,7 +9,7 @@ use crate::{
distribution::metrics::{
ActivityMinimal, ImportConfig, OutputsBase, RealizedMinimal, SupplyCore, UnrealizedBasic,
},
prices,
price,
};
/// TypeCohortMetrics: supply(core), outputs(base), realized(minimal), unrealized(basic).
@@ -59,7 +59,7 @@ impl TypeCohortMetrics {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -73,7 +73,7 @@ impl TypeCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -149,7 +149,7 @@ use crate::{
CohortState, CoreRealizedState, CostBasisData, CostBasisOps, CostBasisRaw,
MinimalRealizedState, RealizedOps, RealizedState, WithCapital, WithoutCapital,
},
prices,
price,
};
pub trait CohortMetricsState {
@@ -270,7 +270,7 @@ pub trait CohortMetricsBase:
/// First phase of computed metrics (indexes from height).
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -10,7 +10,7 @@ use crate::{
internal::{
PerBlock, RatioPerBlock, ValuePerBlock, ValuePerBlockWithDeltas, WindowStartVec, Windows,
},
prices,
price,
};
#[derive(Traversable)]
@@ -115,7 +115,7 @@ impl ProfitabilityBucket {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
is_profit: bool,
exit: &Exit,
@@ -176,7 +176,7 @@ impl ProfitabilityBucket {
pub(crate) fn compute_from_ranges(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
is_profit: bool,
sources: &[&ProfitabilityBucket],
@@ -293,7 +293,7 @@ impl ProfitabilityMetrics {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -16,7 +16,7 @@ use crate::{
FiatPerBlockCumulativeWithSumsAndDeltas, LazyPerBlock, NegCentsUnsignedToDollars,
PerBlockCumulativeRolling, RatioCents64, RollingWindow24hPerBlock, Windows,
},
prices,
price,
};
use crate::distribution::metrics::ImportConfig;
@@ -166,7 +166,7 @@ impl RealizedCore {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
transfer_volume_sum_24h_cents: &impl ReadableVec<Height, Cents>,
@@ -18,7 +18,7 @@ use crate::{
RatioPerBlockStdDevBands, RatioSma, RollingWindows, RollingWindowsFrom1w,
ValuePerBlockCumulativeRolling,
},
prices,
price,
};
use crate::distribution::metrics::ImportConfig;
@@ -206,7 +206,7 @@ impl RealizedFull {
}
#[inline(always)]
pub(crate) fn push_accum(&mut self, accum: &RealizedFullAccum) {
pub(crate) fn push_accum(&mut self, accum: &RealizedFullAccum) -> Cents {
self.cap_raw.push(accum.cap_raw);
self.capitalized.cap_raw.push(accum.capitalized_cap_raw);
@@ -221,6 +221,8 @@ impl RealizedFull {
self.capitalized.price.cents.height.push(capitalized_price);
self.peak_regret.value.block.cents.push(accum.peak_regret());
capitalized_price
}
pub(crate) fn compute_rest_part1(
@@ -240,7 +242,7 @@ impl RealizedFull {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
@@ -13,7 +13,7 @@ use crate::{
FiatPerBlockCumulativeWithSums, FiatPerBlockWithDeltas, Identity, LazyPerBlock,
PriceWithRatioPerBlock,
},
prices,
price,
};
use crate::distribution::metrics::ImportConfig;
@@ -104,7 +104,7 @@ impl RealizedMinimal {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
exit: &Exit,
@@ -3,7 +3,7 @@ use brk_traversable::Traversable;
use brk_types::{Height, Sats, StoredU64, Version};
use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode, WritableVec};
use crate::{indexes, internal::ValuePerBlock, prices};
use crate::{indexes, internal::ValuePerBlock, price};
/// Average amount held per UTXO and per funded address.
///
@@ -53,7 +53,7 @@ impl AvgAmountMetrics {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
supply_sats: &impl ReadableVec<Height, Sats>,
utxo_count: &impl ReadableVec<Height, StoredU64>,
funded_addr_count: &impl ReadableVec<Height, StoredU64>,
@@ -6,7 +6,7 @@ use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVe
use crate::{
distribution::state::{CohortState, CostBasisOps, RealizedOps},
prices,
price,
};
use crate::internal::{
@@ -64,7 +64,7 @@ impl SupplyBase {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -5,7 +5,7 @@ use brk_types::{Height, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode, WritableVec};
use crate::{distribution::state::UnrealizedState, prices};
use crate::{distribution::state::UnrealizedState, price};
use crate::internal::{
HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin, LazyValuePerBlock, ValuePerBlock,
@@ -72,7 +72,7 @@ impl SupplyCore {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -7,7 +7,7 @@ use vecdb::{AnyStoredVec, AnyVec, BytesVec, Exit, ReadableVec, Rw, StorageMode,
use crate::distribution::state::UnrealizedState;
use crate::internal::{CentsSubtractToCentsSigned, FiatPerBlock};
use crate::{distribution::metrics::ImportConfig, prices};
use crate::{distribution::metrics::ImportConfig, price};
use super::UnrealizedCore;
@@ -99,7 +99,7 @@ impl UnrealizedFull {
pub(crate) fn compute_rest_all(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
@@ -13,7 +13,7 @@ use brk_indexer::Lengths;
use brk_types::{Height, Sats};
use vecdb::{Exit, ReadableVec};
use crate::{distribution::state::UnrealizedState, prices};
use crate::{distribution::state::UnrealizedState, price};
pub trait UnrealizedLike: Send + Sync {
fn as_core(&self) -> &UnrealizedCore;
@@ -22,7 +22,7 @@ pub trait UnrealizedLike: Send + Sync {
fn push_state(&mut self, state: &UnrealizedState);
fn compute_rest(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
@@ -46,7 +46,7 @@ impl UnrealizedLike for UnrealizedCore {
}
fn compute_rest(
&mut self,
_prices: &prices::Vecs,
_prices: &price::Vecs,
starting_lengths: &Lengths,
_supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
_supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
@@ -72,7 +72,7 @@ impl UnrealizedLike for UnrealizedFull {
}
fn compute_rest(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
@@ -1,5 +1,6 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_cohort::EntryPrice;
use brk_types::{Cents, SupplyState, Timestamp};
use serde::Serialize;
@@ -8,6 +9,8 @@ pub struct BlockState {
#[serde(flatten)]
pub supply: SupplyState,
#[serde(skip)]
pub entry: EntryPrice,
#[serde(skip)]
pub price: Cents,
#[serde(skip)]
pub timestamp: Timestamp,
+49 -12
View File
@@ -1,6 +1,6 @@
use std::path::{Path, PathBuf};
use brk_cohort::{ByAddrType, Filter};
use brk_cohort::{ByAddrType, EntryPrice, Filter};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
@@ -29,7 +29,7 @@ use crate::{
PerBlockCumulativeRolling, WindowStartVec, Windows, WithAddrTypes,
db_utils::{finalize_db, open_db},
},
outputs, prices, transactions,
outputs, price, transactions,
};
use super::{
@@ -316,7 +316,7 @@ impl Vecs {
outputs: &outputs::Vecs,
transactions: &transactions::Vecs,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;
@@ -341,12 +341,13 @@ impl Vecs {
// Try to resume from checkpoint, fall back to fresh start if needed
let recovered_height = match start_mode {
StartMode::Resume(height) => {
let stamp = Stamp::from(height);
// Roll back only on a reorg. A clean resume has nothing to undo, and an
// interrupted run wrote no rollback metadata (periodic flushes use
// with_changes=false; only the final write creates the `changes/` dir),
// so `rollback_before` would fail with `NotFound`.
let chain_state_rollback = (height < current_height)
.then(|| self.supply_state.rollback_before(Stamp::from(height)));
// Rollback BytesVec state and capture results for validation
let chain_state_rollback = self.supply_state.rollback_before(stamp);
// Validate all rollbacks and imports are consistent
let recovered = recover_state(
height,
chain_state_rollback,
@@ -435,13 +436,34 @@ impl Vecs {
let end = usize::from(recovered_height);
debug!("building supply_state vec for {} heights", recovered_height);
let supply_state_data: Vec<_> = self.supply_state.collect_range_at(0, end);
let capitalized_price_data: Vec<_> = self
.utxo_cohorts
.all
.metrics
.realized
.capitalized
.price
.cents
.height
.collect_range_at(0, end);
let mut entry_anchor = Cents::ZERO;
chain_state = supply_state_data
.into_iter()
.enumerate()
.map(|(h, supply)| BlockState {
supply,
price: self.caches.prices[h],
timestamp: self.caches.timestamps[h],
.map(|(h, supply)| {
let price = self.caches.prices[h];
let entry = EntryPrice::from_is_discount(
entry_anchor == Cents::ZERO || price <= entry_anchor,
);
entry_anchor = capitalized_price_data[h];
BlockState {
supply,
entry,
price,
timestamp: self.caches.timestamps[h],
}
})
.collect();
debug!("chain_state rebuilt");
@@ -473,6 +495,20 @@ impl Vecs {
let prices = std::mem::take(&mut self.caches.prices);
let timestamps = std::mem::take(&mut self.caches.timestamps);
let price_range_max = std::mem::take(&mut self.caches.price_range_max);
let entry_anchor = starting_height
.decremented()
.and_then(|height| {
self.utxo_cohorts
.all
.metrics
.realized
.capitalized
.price
.cents
.height
.collect_one(height)
})
.unwrap_or(Cents::ZERO);
process_blocks(
self,
@@ -485,6 +521,7 @@ impl Vecs {
last_height,
&mut chain_state,
&mut tx_index_to_height,
entry_anchor,
&prices,
&timestamps,
&price_range_max,
+1 -6
View File
@@ -250,12 +250,7 @@ impl Vecs {
identity: LazyVecFrom1::init(
"op_return_index",
version,
indexer
.vecs
.scripts
.op_return
.to_tx_index
.read_only_boxed_clone(),
indexer.vecs.op_return.to_tx_index.read_only_boxed_clone(),
|index, _| index,
),
},
@@ -6,7 +6,7 @@ use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{Database, Exit, Rw, StorageMode};
use crate::{distribution, indexes, prices};
use crate::{distribution, indexes, price};
pub use inner::RarityMeterInner;
@@ -37,7 +37,7 @@ impl RarityMeter {
&mut self,
indexer: &Indexer,
distribution: &distribution::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let realized = &distribution.utxo_cohorts.all.metrics.realized;
@@ -6,6 +6,8 @@ use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::{Vecs, WithInputTypes};
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
@@ -37,6 +39,7 @@ impl Vecs {
let mut itype_cursor = indexer.vecs.inputs.output_type.cursor();
let mut fi_in_cursor = indexer.vecs.transactions.first_txin_index.cursor();
let mut height = skip;
walk_blocks(
&fi_batch,
@@ -65,7 +68,8 @@ impl Vecs {
);
push_block(&mut self.tx_count, agg.txs_all, &agg.txs_per_type);
if self.input_count.all.block.batch_limit_reached() {
height += 1;
if height.is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.input_count.write()?;
self.tx_count.write()?;
@@ -79,12 +83,12 @@ impl Vecs {
self.input_count.write()?;
self.tx_count.write()?;
}
self.input_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
}
self.input_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
for (otype, source) in self.input_count.by_type.iter_typed() {
self.input_share.get_mut(otype).compute_count_ratio(
source,
+2 -3
View File
@@ -3,13 +3,12 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -18,7 +17,7 @@ impl Vecs {
let starting_lengths = indexer.safe_lengths();
self.spent.compute(indexer, exit)?;
self.count.compute(indexer, indexes, blocks, exit)?;
self.count.compute(indexer, blocks, exit)?;
self.per_sec.compute(&self.count, &starting_lengths, exit)?;
self.by_type.compute(indexer, exit)?;
@@ -3,27 +3,25 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let window_starts = blocks.lookback.window_starts();
self.0.compute(
self.sum.compute_count_from_indexes(
starting_height,
&indexes.tx_index.input_count,
&indexer.vecs.transactions.first_tx_index,
&indexes.height.tx_index_count,
&window_starts,
&indexer.vecs.inputs.first_txin_index,
&indexer.vecs.inputs.outpoint,
exit,
0,
)?;
self.compute_rest(starting_height, &window_starts, exit)?;
Ok(())
}
@@ -2,10 +2,7 @@ use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Height;
use schemars::JsonSchema;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw,
StorageMode, VecIndex, VecValue, Version, WritableVec,
};
use vecdb::{Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode, Version};
use crate::{
indexes,
@@ -52,82 +49,19 @@ where
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute<A>(
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
source: &impl ReadableVec<A, T>,
first_indexes: &impl ReadableVec<Height, A>,
count_indexes: &impl ReadableVec<Height, brk_types::StoredU64>,
windows: &WindowStarts<'_>,
exit: &Exit,
skip_count: usize,
) -> Result<()>
where
T: From<f64> + Default + Copy + Ord,
f64: From<T>,
A: VecIndex + VecValue + brk_types::CheckedSub<A>,
{
let combined_version = source.version() + first_indexes.version() + count_indexes.version();
let mut index = max_from;
index = {
self.sum
.validate_computed_version_or_reset(combined_version)?;
index.min(Height::from(self.sum.len()))
};
index = {
self.cumulative
.height
.validate_computed_version_or_reset(combined_version)?;
index.min(Height::from(self.cumulative.height.len()))
};
let start = index.to_usize();
self.sum.truncate_if_needed_at(start)?;
self.cumulative.height.truncate_if_needed_at(start)?;
let mut cumulative_val = index.decremented().map_or(T::from(0_usize), |idx| {
self.cumulative
.height
.collect_one_at(idx.to_usize())
.unwrap_or(T::from(0_usize))
});
let fi_len = first_indexes.len();
let first_indexes_batch: Vec<A> = first_indexes.collect_range_at(start, fi_len);
let count_indexes_batch: Vec<brk_types::StoredU64> =
count_indexes.collect_range_at(start, fi_len);
first_indexes_batch
.into_iter()
.zip(count_indexes_batch)
.try_for_each(|(first_index, count_index)| -> Result<()> {
let count = u64::from(count_index) as usize;
let effective_count = count.saturating_sub(skip_count);
let effective_first_index = first_index + skip_count.min(count);
let efi = effective_first_index.to_usize();
let sum_val = source.fold_range_at(
efi,
efi + effective_count,
T::from(0_usize),
|acc, val| acc + val,
);
self.sum.push(sum_val);
cumulative_val += sum_val;
self.cumulative.height.push(cumulative_val);
Ok(())
})?;
let _lock = exit.lock();
self.sum.write()?;
self.cumulative.height.write()?;
drop(_lock);
self.cumulative
.height
.compute_cumulative(max_from, &self.sum, exit)?;
self.rolling.compute(max_from, windows, &self.sum, exit)?;
Ok(())
}
@@ -6,7 +6,7 @@ use derive_more::{Deref, DerefMut};
use vecdb::{Database, EagerVec, Exit, PcoVec, ReadableVec, Rw, StorageMode};
use crate::internal::{LazyPerBlock, PerBlock, Price};
use crate::{indexes, prices};
use crate::{indexes, price};
use super::{RatioPerBlock, RatioPerBlockPercentiles};
@@ -63,7 +63,7 @@ impl PriceWithRatioPerBlock {
/// Compute price via closure (in cents), then compute ratio.
pub(crate) fn compute_all<F>(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
mut compute_price: F,
@@ -101,7 +101,7 @@ impl PriceWithRatioExtendedPerBlock {
/// Compute ratio and percentiles from already-computed price cents.
pub(crate) fn compute_rest(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -120,7 +120,7 @@ impl PriceWithRatioExtendedPerBlock {
/// Compute price via closure (in cents), then compute ratio and percentiles.
pub(crate) fn compute_all<F>(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
mut compute_price: F,
@@ -10,7 +10,7 @@ use crate::{
CentsUnsignedToDollars, LazyPerBlock, NumericValue, PerBlock, SatsSignedToBitcoin,
SatsToBitcoin, SatsToCents,
},
prices,
price,
};
/// Trait that associates a sats type with its transform to Bitcoin.
@@ -69,7 +69,7 @@ impl ValuePerBlock {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -8,7 +8,7 @@ use vecdb::{
use crate::{
internal::{CentsUnsignedToDollars, SatsToBitcoin, SatsToCents},
prices,
price,
};
/// Raw per-block amount data: sats + cents (stored), btc + usd (lazy), no resolutions.
@@ -44,7 +44,7 @@ impl ValueBlock {
pub(crate) fn compute_cents(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.cents.compute_binary::<Sats, Cents, SatsToCents>(
@@ -6,7 +6,7 @@ use vecdb::{Database, EagerVec, Exit, PcoVec, Rw, StorageMode};
use crate::{
indexes,
internal::{ValueBlock, ValuePerBlock},
prices,
price,
};
#[derive(Traversable)]
@@ -39,7 +39,7 @@ impl ValuePerBlockCumulative {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -61,7 +61,7 @@ impl ValuePerBlockCumulative {
pub(crate) fn compute_with(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
) -> Result<()> {
@@ -10,7 +10,7 @@ use crate::{
LazyRollingAvgsAmountFromHeight, LazyRollingSumsAmountFromHeight, ValuePerBlockCumulative,
WindowStartVec, Windows,
},
prices,
price,
};
#[derive(Deref, DerefMut, Traversable)]
@@ -63,7 +63,7 @@ impl ValuePerBlockCumulativeRolling {
pub(crate) fn compute(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
) -> Result<()> {
@@ -74,7 +74,7 @@ impl ValuePerBlockCumulativeRolling {
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.inner.compute(prices, max_from, exit)
@@ -10,7 +10,7 @@ use crate::{
RollingDistributionValuePerBlock, ValuePerBlockCumulativeRolling, WindowStartVec,
WindowStarts, Windows,
},
prices,
price,
};
#[derive(Deref, DerefMut, Traversable)]
@@ -49,7 +49,7 @@ impl ValuePerBlockFull {
&mut self,
max_from: Height,
windows: &WindowStarts<'_>,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
) -> Result<()> {
@@ -1,60 +0,0 @@
use brk_error::Result;
use brk_indexer::{Indexer, Lengths};
use brk_traversable::Traversable;
use brk_types::TxIndex;
use schemars::JsonSchema;
use vecdb::{Database, Exit, LazyVecFrom2, ReadableVec, Rw, StorageMode, Version};
use crate::{
indexes,
internal::{ComputedVecValue, NumericValue, TxDerivedDistribution},
};
#[derive(Traversable)]
pub struct LazyPerTxDistribution<T, S1, S2, M: StorageMode = Rw>
where
T: ComputedVecValue + PartialOrd + JsonSchema,
S1: ComputedVecValue,
S2: ComputedVecValue,
{
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
#[traversable(flatten)]
pub distribution: TxDerivedDistribution<T, M>,
}
impl<T, S1, S2> LazyPerTxDistribution<T, S1, S2>
where
T: NumericValue + JsonSchema,
S1: ComputedVecValue + JsonSchema,
S2: ComputedVecValue + JsonSchema,
{
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
) -> Result<Self> {
let distribution = TxDerivedDistribution::forced_import(db, name, version, indexes)?;
Ok(Self {
tx_index,
distribution,
})
}
pub(crate) fn derive_from(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()>
where
T: Copy + Ord + From<f64> + Default,
f64: From<T>,
LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>: ReadableVec<TxIndex, T>,
{
self.distribution
.derive_from(indexer, indexes, starting_lengths, &self.tx_index, exit)
}
}
@@ -1,36 +1,33 @@
use brk_traversable::Traversable;
use brk_types::{TxIndex, Version};
use schemars::JsonSchema;
use vecdb::{LazyVecFrom2, UnaryTransform};
use vecdb::{LazyVecFrom1, UnaryTransform};
use crate::internal::{ComputedVecValue, LazyTxDerivedDistribution, TxDerivedDistribution};
/// Like `LazyPerTxDistribution` but with a lazy-derived distribution
/// (transformed from another type's distribution rather than eagerly computed).
/// Per-transaction lazy values with a distribution transformed from another type's distribution.
#[derive(Clone, Traversable)]
pub struct LazyPerTxDistributionTransformed<T, S1, S2, DSource>
pub struct LazyPerTxDistributionTransformed<T, S, DSource>
where
T: ComputedVecValue + JsonSchema,
S1: ComputedVecValue,
S2: ComputedVecValue,
S: ComputedVecValue,
DSource: ComputedVecValue,
{
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
pub tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
#[traversable(flatten)]
pub distribution: LazyTxDerivedDistribution<T, DSource>,
}
impl<T, S1, S2, DSource> LazyPerTxDistributionTransformed<T, S1, S2, DSource>
impl<T, S, DSource> LazyPerTxDistributionTransformed<T, S, DSource>
where
T: ComputedVecValue + JsonSchema + 'static,
S1: ComputedVecValue + JsonSchema,
S2: ComputedVecValue + JsonSchema,
S: ComputedVecValue + JsonSchema,
DSource: ComputedVecValue + JsonSchema,
{
pub(crate) fn new<F: UnaryTransform<DSource, T>>(
name: &str,
version: Version,
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
source_distribution: &TxDerivedDistribution<DSource>,
) -> Self {
let distribution =
@@ -1,11 +1,9 @@
mod derived;
mod distribution;
mod lazy_derived;
mod lazy_distribution;
mod lazy_distribution_transformed;
pub use derived::*;
pub use distribution::*;
pub use lazy_derived::*;
pub use lazy_distribution::*;
pub use lazy_distribution_transformed::*;
@@ -146,11 +146,11 @@ impl UnaryTransform<StoredU64, Weight> for VBytesToWeight {
}
}
pub struct VSizeToWeight;
pub struct WeightToVSize;
impl UnaryTransform<VSize, Weight> for VSizeToWeight {
impl UnaryTransform<Weight, VSize> for WeightToVSize {
#[inline(always)]
fn apply(vsize: VSize) -> Weight {
Weight::from(vsize)
fn apply(weight: Weight) -> VSize {
VSize::from(weight)
}
}
@@ -8,7 +8,7 @@ mod specialized;
pub use arithmetic::{
BlocksToDaysF32, DifficultyToHashF64, HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin,
Identity, MaskSats, OneMinusBp16, OneMinusF64, ReturnF32Tenths, ReturnI8, ReturnU16,
ThsToPhsF32, VBytesToWeight, VSizeToWeight,
ThsToPhsF32, VBytesToWeight, WeightToVSize,
};
pub use bps::{
Bp16ToFloat, Bp16ToPercent, Bp32ToFloat, Bp32ToPercent, Bps16ToFloat, Bps16ToPercent,
@@ -11,7 +11,7 @@ use rayon::prelude::*;
use schemars::JsonSchema;
use vecdb::{AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, WritableVec};
use crate::{indexes, prices};
use crate::{indexes, price};
use super::{
BpsType, NumericValue, PerBlock, PerBlockCumulativeRolling, PercentPerBlock, ValuePerBlock,
@@ -229,7 +229,7 @@ impl WithAddrTypes<ValuePerBlock> {
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.all.compute(prices, max_from, exit)?;
+2 -2
View File
@@ -4,7 +4,7 @@ use brk_types::{BasisPointsSigned32, Bitcoin, Cents, Date, Day1, Dollars, Sats};
use vecdb::{AnyVec, Exit, ReadableOptionVec, ReadableVec, VecIndex};
use super::{ByDcaPeriod, Vecs};
use crate::{blocks, indexes, internal::RatioDiffCentsBps32, market, prices};
use crate::{blocks, indexes, internal::RatioDiffCentsBps32, market, price};
const DCA_AMOUNT: Dollars = Dollars::mint(100.0);
@@ -13,7 +13,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
lookback: &market::lookback::Vecs,
exit: &Exit,
+60 -35
View File
@@ -20,9 +20,10 @@ mod internal;
mod investing;
mod market;
mod mining;
mod op_return;
mod outputs;
mod pools;
pub mod prices;
pub mod price;
mod supply;
mod transactions;
@@ -38,15 +39,16 @@ pub struct Computer<M: StorageMode = Rw> {
pub investing: Box<investing::Vecs<M>>,
pub market: Box<market::Vecs<M>>,
pub pools: Box<pools::Vecs<M>>,
pub prices: Box<prices::Vecs<M>>,
pub price: Box<price::Vecs<M>>,
#[traversable(flatten)]
pub distribution: Box<distribution::Vecs<M>>,
pub supply: Box<supply::Vecs<M>>,
pub inputs: Box<inputs::Vecs<M>>,
pub outputs: Box<outputs::Vecs<M>>,
pub op_return: Box<op_return::Vecs<M>>,
}
const VERSION: Version = Version::new(6);
const VERSION: Version = Version::new(7);
impl Computer {
pub fn forced_import(outputs_path: &Path, indexer: &Indexer) -> Result<Self> {
@@ -66,14 +68,14 @@ impl Computer {
)?))
})?;
let (constants, prices) = timed("Imported prices/constants", || -> Result<_> {
let (constants, price) = timed("Imported price/constants", || -> Result<_> {
let constants = Box::new(constants::Vecs::new(VERSION, &indexes));
let prices = Box::new(prices::Vecs::forced_import(
let price = Box::new(price::Vecs::forced_import(
&computed_path,
VERSION,
&indexes,
)?);
Ok((constants, prices))
Ok((constants, price))
})?;
let blocks = timed("Imported blocks", || -> Result<_> {
@@ -87,8 +89,9 @@ impl Computer {
let cached_starts = blocks.lookback.cached_window_starts();
let (inputs, outputs, mining, transactions, pools, cointime) =
timed("Imported inputs/outputs/mining/tx/pools/cointime", || {
let (inputs, outputs, mining, transactions, pools, cointime, op_return) = timed(
"Imported inputs/outputs/mining/tx/pools/cointime/op_return",
|| {
thread::scope(|s| -> Result<_> {
let inputs_handle = big_thread().spawn_scoped(s, || -> Result<_> {
Ok(Box::new(inputs::Vecs::forced_import(
@@ -143,15 +146,34 @@ impl Computer {
&cached_starts,
)?);
let op_return_handle = big_thread().spawn_scoped(s, || -> Result<_> {
Ok(Box::new(op_return::Vecs::forced_import(
&computed_path,
VERSION,
&indexes,
&cached_starts,
)?))
})?;
let inputs = inputs_handle.join().unwrap()?;
let outputs = outputs_handle.join().unwrap()?;
let mining = mining_handle.join().unwrap()?;
let transactions = transactions_handle.join().unwrap()?;
let pools = pools_handle.join().unwrap()?;
let op_return = op_return_handle.join().unwrap()?;
Ok((inputs, outputs, mining, transactions, pools, cointime))
Ok((
inputs,
outputs,
mining,
transactions,
pools,
cointime,
op_return,
))
})
})?;
},
)?;
// Market, indicators, and distribution are independent; import in parallel.
// Supply depends on distribution so it runs after.
@@ -223,8 +245,9 @@ impl Computer {
cointime,
indexes,
inputs,
prices,
price,
outputs,
op_return,
};
Self::retain_databases(&computed_path)?;
@@ -244,11 +267,12 @@ impl Computer {
investing::DB_NAME,
market::DB_NAME,
pools::DB_NAME,
prices::DB_NAME,
price::DB_NAME,
distribution::DB_NAME,
supply::DB_NAME,
inputs::DB_NAME,
outputs::DB_NAME,
op_return::DB_NAME,
];
if !computed_path.exists() {
@@ -292,13 +316,12 @@ impl Computer {
let (inputs_result, prices_result) = rayon::join(
|| {
timed("Computed inputs", || {
self.inputs
.compute(indexer, &self.indexes, &self.blocks, exit)
self.inputs.compute(indexer, &self.blocks, exit)
})
},
|| {
timed("Computed prices", || {
self.prices.compute(indexer, &self.indexes, exit)
timed("Computed price", || {
self.price.compute(indexer, &self.indexes, exit)
})
},
);
@@ -310,7 +333,7 @@ impl Computer {
let market = scope.spawn(|| {
timed("Computed market", || {
self.market
.compute(indexer, &self.prices, &self.indexes, &self.blocks, exit)
.compute(indexer, &self.price, &self.indexes, &self.blocks, exit)
})
});
@@ -321,7 +344,7 @@ impl Computer {
&self.indexes,
&self.blocks,
&self.inputs,
&self.prices,
&self.price,
exit,
)
})?;
@@ -331,24 +354,25 @@ impl Computer {
&self.indexes,
&self.blocks,
&self.transactions,
&self.prices,
&self.price,
exit,
)
})
});
let op_return = scope.spawn(|| {
timed("Computed OP_RETURN", || {
self.op_return.compute(indexer, exit)
})
});
timed("Computed outputs", || {
self.outputs.compute(
indexer,
&self.indexes,
&self.inputs,
&self.blocks,
&self.prices,
exit,
)
self.outputs
.compute(indexer, &self.inputs, &self.blocks, &self.price, exit)
})?;
tx_mining.join().unwrap()?;
op_return.join().unwrap()?;
market.join().unwrap()?;
Ok(())
})?;
@@ -360,7 +384,7 @@ impl Computer {
indexer,
&self.indexes,
&self.blocks,
&self.prices,
&self.price,
&self.mining,
exit,
)
@@ -372,7 +396,7 @@ impl Computer {
self.investing.compute(
indexer,
&self.indexes,
&self.prices,
&self.price,
&self.blocks,
&self.market.lookback,
exit,
@@ -388,7 +412,7 @@ impl Computer {
&self.outputs,
&self.transactions,
&self.blocks,
&self.prices,
&self.price,
exit,
)
})?;
@@ -421,7 +445,7 @@ impl Computer {
&self.blocks,
&self.mining,
&self.transactions,
&self.prices,
&self.price,
&self.distribution,
exit,
)
@@ -430,7 +454,7 @@ impl Computer {
timed("Computed cointime", || {
self.cointime.compute(
indexer,
&self.prices,
&self.price,
&self.blocks,
&self.mining,
&self.supply,
@@ -445,7 +469,7 @@ impl Computer {
self.indicators
.rarity_meter
.compute(indexer, &self.distribution, &self.prices, exit)?;
.compute(indexer, &self.distribution, &self.price, exit)?;
info!("Total compute time: {:?}", compute_start.elapsed());
Ok(())
@@ -498,11 +522,12 @@ impl_iter_named!(
investing,
market,
pools,
prices,
price,
distribution,
supply,
inputs,
outputs
outputs,
op_return
);
fn timed<T>(label: &str, f: impl FnOnce() -> T) -> T {
@@ -4,13 +4,13 @@ use brk_types::{StoredF32, Timestamp};
use vecdb::{Exit, ReadableVec, VecIndex};
use super::Vecs;
use crate::{indexes, prices};
use crate::{indexes, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
indexes: &indexes::Vecs,
exit: &Exit,
) -> Result<()> {
+2 -2
View File
@@ -2,7 +2,7 @@ use brk_error::Result;
use brk_indexer::Indexer;
use vecdb::Exit;
use crate::{blocks, indexes, prices};
use crate::{blocks, indexes, price};
use super::Vecs;
@@ -10,7 +10,7 @@ impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, prices};
use crate::{blocks, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, prices};
use crate::{blocks, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
@@ -4,13 +4,13 @@ use brk_types::{BasisPoints16, StoredF32};
use vecdb::{Exit, ReadableVec, VecIndex};
use super::Vecs;
use crate::{blocks, prices};
use crate::{blocks, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -5,14 +5,14 @@ use vecdb::Exit;
use super::Vecs;
use crate::{
blocks, internal::RatioDiffDollarsBps32, investing::ByDcaPeriod, market::lookback, prices,
blocks, internal::RatioDiffDollarsBps32, investing::ByDcaPeriod, market::lookback, price,
};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
lookback: &lookback::Vecs,
exit: &Exit,
@@ -10,7 +10,7 @@ use super::{
use crate::{
blocks,
internal::{RatioDollarsBp32, WindowsTo1m},
prices,
price,
};
impl Vecs {
@@ -19,7 +19,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
returns: &returns::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
moving_average: &moving_average::Vecs,
exit: &Exit,
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::MacdChain;
use crate::{blocks, prices};
use crate::{blocks, price};
#[allow(clippy::too_many_arguments)]
pub(super) fn compute(
chain: &mut MacdChain,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
fast_days: usize,
slow_days: usize,
signal_days: usize,
+2 -2
View File
@@ -3,7 +3,7 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes, prices, transactions};
use crate::{blocks, indexes, price, transactions};
impl Vecs {
#[allow(clippy::too_many_arguments)]
@@ -13,7 +13,7 @@ impl Vecs {
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
transactions: &transactions::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;

Some files were not shown because too many files have changed in this diff Show More