mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-22 12:23:04 -07:00
Compare commits
59 Commits
v0.3.0-beta.11
...
v0.3.2
| Author | SHA1 | Date | |
|---|---|---|---|
| 744032f1f1 | |||
| 99b171bad6 | |||
| 37e2b6eae2 | |||
| a967fe8f35 | |||
| a3f3c54675 | |||
| f41874f438 | |||
| 98bbfec525 | |||
| 1bcf3235b6 | |||
| 07734b8bab | |||
| a2fd1e03ad | |||
| 90e8741fb7 | |||
| 5f5563fece | |||
| c7edfce481 | |||
| 7b3dd83b93 | |||
| cae16227fd | |||
| dc2ca0ca27 | |||
| d161462137 | |||
| be20633945 | |||
| 2bbc535b58 | |||
| 88c38e74f9 | |||
| a61b76a4a5 | |||
| 46b888337c | |||
| 4b49a04186 | |||
| 15b0cd2445 | |||
| 76720434d7 | |||
| 200cd1011e | |||
| cb9f277d49 | |||
| 102933b406 | |||
| e64ffac8d1 | |||
| a94d31dfdf | |||
| 087a3b6fd6 | |||
| 7181d59966 | |||
| 3b7734a61a | |||
| 7860c5a8bd | |||
| 5df399d2f7 | |||
| b2345db279 | |||
| 7e2fc8b455 | |||
| c1ff095e4b | |||
| cc8fde59e8 | |||
| e43b53b429 | |||
| 6938204a24 | |||
| 52883bbdba | |||
| 100495fdba | |||
| 0ad5be6974 | |||
| 66037c862f | |||
| ee20175cbf | |||
| 7ad0adf659 | |||
| 6219d2301d | |||
| 0aaffc6c43 | |||
| 9c74881c5d | |||
| bf8de73541 | |||
| 56e8103178 | |||
| 773c0d090b | |||
| d6f4c0ac19 | |||
| 0552ba60d2 | |||
| ff056587f7 | |||
| 0b871e8600 | |||
| 0bdca9086a | |||
| bbab864ed9 |
@@ -1,15 +0,0 @@
|
||||
name: Check outdated dependencies
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 9 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
outdated:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- run: cargo install cargo-outdated
|
||||
- run: cargo outdated --exit-code 1 --depth 1
|
||||
@@ -1,296 +0,0 @@
|
||||
# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
|
||||
#
|
||||
# Copyright 2022-2024, axodotdev
|
||||
# SPDX-License-Identifier: MIT or Apache-2.0
|
||||
#
|
||||
# CI that:
|
||||
#
|
||||
# * checks for a Git Tag that looks like a release
|
||||
# * builds artifacts with dist (archives, installers, hashes)
|
||||
# * uploads those artifacts to temporary workflow zip
|
||||
# * on success, uploads the artifacts to a GitHub Release
|
||||
#
|
||||
# Note that the GitHub Release will be created with a generated
|
||||
# title/body based on your changelogs.
|
||||
|
||||
name: Release
|
||||
permissions:
|
||||
"contents": "write"
|
||||
|
||||
# This task will run whenever you push a git tag that looks like a version
|
||||
# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
|
||||
# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
|
||||
# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
|
||||
# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
|
||||
#
|
||||
# If PACKAGE_NAME is specified, then the announcement will be for that
|
||||
# package (erroring out if it doesn't have the given version or isn't dist-able).
|
||||
#
|
||||
# If PACKAGE_NAME isn't specified, then the announcement will be for all
|
||||
# (dist-able) packages in the workspace with that version (this mode is
|
||||
# intended for workspaces with only one dist-able package, or with all dist-able
|
||||
# packages versioned/released in lockstep).
|
||||
#
|
||||
# If you push multiple tags at once, separate instances of this workflow will
|
||||
# spin up, creating an independent announcement for each one. However, GitHub
|
||||
# will hard limit this to 3 tags per commit, as it will assume more tags is a
|
||||
# mistake.
|
||||
#
|
||||
# If there's a prerelease-style suffix to the version, then the release(s)
|
||||
# will be marked as a prerelease.
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
tags:
|
||||
- '**[0-9]+.[0-9]+.[0-9]+*'
|
||||
|
||||
jobs:
|
||||
# Run 'dist plan' (or host) to determine what tasks we need to do
|
||||
plan:
|
||||
runs-on: "ubuntu-22.04"
|
||||
outputs:
|
||||
val: ${{ steps.plan.outputs.manifest }}
|
||||
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
|
||||
tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
|
||||
publishing: ${{ !github.event.pull_request }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install dist
|
||||
# we specify bash to get pipefail; it guards against the `curl` command
|
||||
# failing. otherwise `sh` won't catch that `curl` returned non-0
|
||||
shell: bash
|
||||
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.2/cargo-dist-installer.sh | sh"
|
||||
- name: Cache dist
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/dist
|
||||
# sure would be cool if github gave us proper conditionals...
|
||||
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
|
||||
# functionality based on whether this is a pull_request, and whether it's from a fork.
|
||||
# (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
|
||||
# but also really annoying to build CI around when it needs secrets to work right.)
|
||||
- id: plan
|
||||
run: |
|
||||
dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
cat plan-dist-manifest.json
|
||||
echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
- name: "Upload dist-manifest.json"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-plan-dist-manifest
|
||||
path: plan-dist-manifest.json
|
||||
|
||||
# Build and packages all the platform-specific things
|
||||
build-local-artifacts:
|
||||
name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
|
||||
# Let the initial task tell us to not run (currently very blunt)
|
||||
needs:
|
||||
- plan
|
||||
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# Target platforms/runners are computed by dist in create-release.
|
||||
# Each member of the matrix has the following arguments:
|
||||
#
|
||||
# - runner: the github runner
|
||||
# - dist-args: cli flags to pass to dist
|
||||
# - install-dist: expression to run to install dist on the runner
|
||||
#
|
||||
# Typically there will be:
|
||||
# - 1 "global" task that builds universal installers
|
||||
# - N "local" tasks that build each platform's binaries and platform-specific installers
|
||||
matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.container && matrix.container.image || null }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
|
||||
steps:
|
||||
- name: enable windows longpaths
|
||||
run: |
|
||||
git config --global core.longpaths true
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install Rust non-interactively if not already installed
|
||||
if: ${{ matrix.container }}
|
||||
run: |
|
||||
if ! command -v cargo > /dev/null 2>&1; then
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
fi
|
||||
- name: Install dist
|
||||
run: ${{ matrix.install_dist.run }}
|
||||
# Get the dist-manifest
|
||||
- name: Fetch local artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
${{ matrix.packages_install }}
|
||||
- name: Build artifacts
|
||||
run: |
|
||||
# Actually do builds and make zips and whatnot
|
||||
dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
- id: cargo-dist
|
||||
name: Post-build
|
||||
# We force bash here just because github makes it really hard to get values up
|
||||
# to "real" actions without writing to env-vars, and writing to env-vars has
|
||||
# inconsistent syntax between shell and powershell.
|
||||
shell: bash
|
||||
run: |
|
||||
# Parse out what we just built and upload it to scratch storage
|
||||
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
|
||||
dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
|
||||
- name: "Upload artifacts"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-build-local-${{ join(matrix.targets, '_') }}
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
|
||||
# Build and package all the platform-agnostic(ish) things
|
||||
build-global-artifacts:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/
|
||||
- run: chmod +x ~/.cargo/bin/dist
|
||||
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
|
||||
- name: Fetch local artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- id: cargo-dist
|
||||
shell: bash
|
||||
run: |
|
||||
dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
|
||||
# Parse out what we just built and upload it to scratch storage
|
||||
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
|
||||
jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
|
||||
- name: "Upload artifacts"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-build-global
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
# Determines if we should publish/announce
|
||||
host:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
- build-global-artifacts
|
||||
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
outputs:
|
||||
val: ${{ steps.host.outputs.manifest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/
|
||||
- run: chmod +x ~/.cargo/bin/dist
|
||||
# Fetch artifacts from scratch-storage
|
||||
- name: Fetch artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- id: host
|
||||
shell: bash
|
||||
run: |
|
||||
dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
|
||||
echo "artifacts uploaded and released successfully"
|
||||
cat dist-manifest.json
|
||||
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
- name: "Upload dist-manifest.json"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
# Overwrite the previous copy
|
||||
name: artifacts-dist-manifest
|
||||
path: dist-manifest.json
|
||||
# Create a GitHub Release while uploading all files to it
|
||||
- name: "Download GitHub Artifacts"
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: artifacts
|
||||
merge-multiple: true
|
||||
- name: Cleanup
|
||||
run: |
|
||||
# Remove the granular manifests
|
||||
rm -f artifacts/*-dist-manifest.json
|
||||
- name: Create GitHub Release
|
||||
env:
|
||||
PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
|
||||
ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}"
|
||||
ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}"
|
||||
RELEASE_COMMIT: "${{ github.sha }}"
|
||||
run: |
|
||||
# Write and read notes from a file to avoid quoting breaking things
|
||||
echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt
|
||||
|
||||
gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
|
||||
|
||||
announce:
|
||||
needs:
|
||||
- plan
|
||||
- host
|
||||
# use "always() && ..." to allow us to wait for all publish jobs while
|
||||
# still allowing individual publish jobs to skip themselves (for prereleases).
|
||||
# "host" however must run to completion, no skipping allowed!
|
||||
if: ${{ always() && needs.host.result == 'success' }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
Generated
+123
-134
@@ -103,9 +103,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.0"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
@@ -176,11 +176,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "base58ck"
|
||||
version = "0.1.0"
|
||||
version = "0.1.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
|
||||
checksum = "ec5dc7e09f7bb15f0062da7c03086d6b71a2c84e0af4fccbbc7d8c6559847816"
|
||||
dependencies = [
|
||||
"bitcoin-internals",
|
||||
"bitcoin_hashes",
|
||||
]
|
||||
|
||||
@@ -222,20 +221,19 @@ dependencies = [
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"shlex 1.3.0",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin"
|
||||
version = "0.32.9"
|
||||
version = "0.32.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9cf93e61f2dbc3e3c41234ca26a65e2c0b0975c52e0f069ab9893ebbede584d3"
|
||||
checksum = "39581299241111285f3268ba75ddf372746fd041620918b145c1af9d75e91b6c"
|
||||
dependencies = [
|
||||
"base58ck",
|
||||
"base64 0.21.7",
|
||||
"bech32",
|
||||
"bitcoin-internals",
|
||||
"bitcoin-io",
|
||||
"bitcoin-units",
|
||||
"bitcoin_hashes",
|
||||
@@ -245,36 +243,26 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin-internals"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin-io"
|
||||
version = "0.1.4"
|
||||
version = "0.1.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953"
|
||||
checksum = "11301df0b06f22dea7bb1916403fdd88a371031e495c49b8f96931b28189e175"
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin-units"
|
||||
version = "0.1.3"
|
||||
version = "0.1.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "346568ebaab2918487cea76dd55dae13c27bb618cdb737c952e69eb2017c4118"
|
||||
checksum = "57bad157b78d0d1b22c4cbb6a35a566211fc4d14866a37f2c780652b50f3b845"
|
||||
dependencies = [
|
||||
"bitcoin-internals",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin_hashes"
|
||||
version = "0.14.1"
|
||||
version = "0.14.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b"
|
||||
checksum = "0c9901a56e133a1fc86eeb1113e2591f45f4682451ca893bff494d2f88918e3f"
|
||||
dependencies = [
|
||||
"bitcoin-io",
|
||||
"hex-conservative",
|
||||
@@ -295,7 +283,7 @@ checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
|
||||
|
||||
[[package]]
|
||||
name = "blk"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_error",
|
||||
@@ -308,7 +296,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_bencher",
|
||||
"brk_bindgen",
|
||||
@@ -333,7 +321,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_alloc"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
"mimalloc",
|
||||
@@ -341,7 +329,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_bencher"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_error",
|
||||
"brk_logger",
|
||||
@@ -351,14 +339,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_bencher_visualizer"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"plotters",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brk_bindgen"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_cohort",
|
||||
"brk_query",
|
||||
@@ -371,7 +359,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_cli"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"brk_alloc",
|
||||
@@ -396,7 +384,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_client"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_cohort",
|
||||
"brk_types",
|
||||
@@ -407,7 +395,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_cohort"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_error",
|
||||
"brk_traversable",
|
||||
@@ -419,7 +407,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_computer"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_alloc",
|
||||
@@ -448,7 +436,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_error"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"fjall",
|
||||
@@ -464,7 +452,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_fetcher"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_error",
|
||||
"brk_logger",
|
||||
@@ -476,7 +464,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_indexer"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_alloc",
|
||||
@@ -502,7 +490,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_iterator"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_error",
|
||||
@@ -513,7 +501,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_logger"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"jiff",
|
||||
"owo-colors",
|
||||
@@ -524,7 +512,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_mempool"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_error",
|
||||
@@ -540,7 +528,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_oracle"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_indexer",
|
||||
"brk_types",
|
||||
@@ -550,7 +538,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_query"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_computer",
|
||||
@@ -575,7 +563,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_reader"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_error",
|
||||
@@ -591,7 +579,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_rpc"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_error",
|
||||
@@ -608,7 +596,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_server"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"aide",
|
||||
"axum",
|
||||
@@ -618,6 +606,7 @@ dependencies = [
|
||||
"brk_indexer",
|
||||
"brk_logger",
|
||||
"brk_mempool",
|
||||
"brk_oracle",
|
||||
"brk_query",
|
||||
"brk_reader",
|
||||
"brk_rpc",
|
||||
@@ -640,7 +629,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_store"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_error",
|
||||
"brk_types",
|
||||
@@ -651,7 +640,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_traversable"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_traversable_derive",
|
||||
"brk_types",
|
||||
@@ -664,7 +653,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_traversable_derive"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -673,7 +662,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_types"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"brk_error",
|
||||
@@ -696,7 +685,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_website"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"brk_logger",
|
||||
@@ -711,9 +700,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "8.0.2"
|
||||
version = "8.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560"
|
||||
checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
@@ -722,9 +711,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brotli-decompressor"
|
||||
version = "5.0.0"
|
||||
version = "5.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
|
||||
checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
@@ -732,9 +721,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.2"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
@@ -768,14 +757,14 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.62"
|
||||
version = "1.2.63"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
|
||||
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
"libc",
|
||||
"shlex",
|
||||
"shlex 2.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -968,9 +957,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "corepc-types"
|
||||
version = "0.13.0"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b96c7869aa8234d10a41cbe3a1697bcb3a2482c48d9eb3541b3a4014a81afdad"
|
||||
checksum = "f095534efdb8f2f43d48b9c3e9f35aefdf29ec6a5f1895064f575a67bc2a8dfe"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"serde",
|
||||
@@ -1066,9 +1055,9 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "6.1.0"
|
||||
version = "6.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
|
||||
checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
@@ -1133,9 +1122,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1186,9 +1175,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
||||
|
||||
[[package]]
|
||||
name = "enum_dispatch"
|
||||
@@ -1251,9 +1240,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "fjall"
|
||||
version = "3.0.4"
|
||||
version = "3.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ebf22b812878dcd767879cb19e03124fd62563dce6410f96538175fba0c132d"
|
||||
checksum = "b62b25b4d815ae178d7d9e4aa32ee59f072efd5431c736abede1e6ee13c8c453"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"byteview",
|
||||
@@ -1261,7 +1250,7 @@ dependencies = [
|
||||
"flume",
|
||||
"log",
|
||||
"lsm-tree",
|
||||
"lz4_flex 0.11.6",
|
||||
"lz4_flex",
|
||||
"tempfile",
|
||||
"xxhash-rust",
|
||||
]
|
||||
@@ -1533,9 +1522,9 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
|
||||
checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
@@ -1578,9 +1567,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.9.0"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
|
||||
checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -1849,23 +1838,23 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jiff"
|
||||
version = "0.2.24"
|
||||
version = "0.2.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d"
|
||||
checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102"
|
||||
dependencies = [
|
||||
"jiff-static",
|
||||
"log",
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde_core",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jiff-static"
|
||||
version = "0.2.24"
|
||||
version = "0.2.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7"
|
||||
checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1890,9 +1879,9 @@ checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.98"
|
||||
version = "0.3.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
|
||||
checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
@@ -1947,9 +1936,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.47"
|
||||
version = "0.1.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
|
||||
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
@@ -1968,9 +1957,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.16"
|
||||
version = "0.1.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
|
||||
checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
@@ -2004,15 +1993,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
version = "0.4.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
|
||||
|
||||
[[package]]
|
||||
name = "lsm-tree"
|
||||
version = "3.0.4"
|
||||
version = "3.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9bfd2a6ea0c1d430c13643002f35800a87f200fc8ac4827f18a2db9d9fd0644"
|
||||
checksum = "e447ac67ff6aef4ec07fc19e507b219336cbba90a697c0dbeb1bf51b91536b67"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"byteview",
|
||||
@@ -2020,7 +2009,7 @@ dependencies = [
|
||||
"enum_dispatch",
|
||||
"interval-heap",
|
||||
"log",
|
||||
"lz4_flex 0.11.6",
|
||||
"lz4_flex",
|
||||
"quick_cache",
|
||||
"rustc-hash",
|
||||
"self_cell",
|
||||
@@ -2030,20 +2019,14 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
|
||||
dependencies = [
|
||||
"twox-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
|
||||
dependencies = [
|
||||
"twox-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
@@ -2053,9 +2036,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
version = "2.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
@@ -2068,9 +2051,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.50"
|
||||
version = "0.1.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640"
|
||||
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
@@ -2099,9 +2082,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.2.0"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
|
||||
checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi",
|
||||
@@ -2110,7 +2093,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mmpl"
|
||||
version = "0.3.0-beta.11"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"brk_error",
|
||||
"brk_mempool",
|
||||
@@ -2133,9 +2116,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.1"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
||||
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
@@ -2387,9 +2370,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quickmatch"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "848244615004bddb7273545dfe909ead495ed734f9faf130c43a7daccca2bf99"
|
||||
checksum = "d6abd98fde5c9aa23590316caedc9e90be29da035d33905cdae71a54f034aaac"
|
||||
dependencies = [
|
||||
"rustc-hash",
|
||||
]
|
||||
@@ -2773,9 +2756,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
@@ -2856,6 +2839,12 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.9"
|
||||
@@ -2876,9 +2865,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.3"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -3150,9 +3139,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tower-http"
|
||||
version = "0.6.10"
|
||||
version = "0.6.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"
|
||||
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags 2.11.1",
|
||||
@@ -3266,9 +3255,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.13.2"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
@@ -3360,7 +3349,7 @@ dependencies = [
|
||||
"itoa",
|
||||
"libc",
|
||||
"log",
|
||||
"lz4_flex 0.13.1",
|
||||
"lz4_flex",
|
||||
"parking_lot",
|
||||
"pco",
|
||||
"rawdb",
|
||||
@@ -3426,9 +3415,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
|
||||
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
@@ -3439,9 +3428,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
|
||||
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -3449,9 +3438,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
|
||||
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
@@ -3462,9 +3451,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
|
||||
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -3505,9 +3494,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.98"
|
||||
version = "0.3.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
|
||||
checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
@@ -3857,18 +3846,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.48"
|
||||
version = "0.8.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
|
||||
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.48"
|
||||
version = "0.8.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
|
||||
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
+30
-40
@@ -4,7 +4,7 @@ members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.3.0-beta.11"
|
||||
package.version = "0.3.2"
|
||||
package.homepage = "https://bitcoinresearchkit.org"
|
||||
package.repository = "https://github.com/bitcoinresearchkit/brk"
|
||||
package.readme = "README.md"
|
||||
@@ -28,9 +28,6 @@ lto = false
|
||||
strip = false
|
||||
inherits = "release"
|
||||
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
|
||||
[profile.profiling]
|
||||
inherits = "release"
|
||||
debug = true
|
||||
@@ -38,38 +35,38 @@ debug = true
|
||||
[workspace.dependencies]
|
||||
aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
|
||||
bitcoin = { version = "0.32.9", features = ["serde"] }
|
||||
brk_alloc = { version = "0.3.0-beta.11", path = "crates/brk_alloc" }
|
||||
brk_bencher = { version = "0.3.0-beta.11", path = "crates/brk_bencher" }
|
||||
brk_bindgen = { version = "0.3.0-beta.11", path = "crates/brk_bindgen" }
|
||||
brk_cli = { version = "0.3.0-beta.11", path = "crates/brk_cli" }
|
||||
brk_client = { version = "0.3.0-beta.11", path = "crates/brk_client" }
|
||||
brk_cohort = { version = "0.3.0-beta.11", path = "crates/brk_cohort" }
|
||||
brk_computer = { version = "0.3.0-beta.11", path = "crates/brk_computer" }
|
||||
brk_error = { version = "0.3.0-beta.11", path = "crates/brk_error" }
|
||||
brk_fetcher = { version = "0.3.0-beta.11", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.3.0-beta.11", path = "crates/brk_indexer" }
|
||||
brk_iterator = { version = "0.3.0-beta.11", path = "crates/brk_iterator" }
|
||||
brk_logger = { version = "0.3.0-beta.11", path = "crates/brk_logger" }
|
||||
brk_mempool = { version = "0.3.0-beta.11", path = "crates/brk_mempool" }
|
||||
brk_oracle = { version = "0.3.0-beta.11", path = "crates/brk_oracle" }
|
||||
brk_query = { version = "0.3.0-beta.11", path = "crates/brk_query", features = ["tokio"] }
|
||||
brk_reader = { version = "0.3.0-beta.11", path = "crates/brk_reader" }
|
||||
brk_rpc = { version = "0.3.0-beta.11", path = "crates/brk_rpc" }
|
||||
brk_server = { version = "0.3.0-beta.11", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.3.0-beta.11", path = "crates/brk_store" }
|
||||
brk_traversable = { version = "0.3.0-beta.11", path = "crates/brk_traversable", features = ["pco", "derive"] }
|
||||
brk_traversable_derive = { version = "0.3.0-beta.11", path = "crates/brk_traversable_derive" }
|
||||
brk_types = { version = "0.3.0-beta.11", path = "crates/brk_types" }
|
||||
brk_website = { version = "0.3.0-beta.11", path = "crates/brk_website" }
|
||||
bitcoin = { version = "0.32.100", features = ["serde"] }
|
||||
brk_alloc = { version = "0.3.2", path = "crates/brk_alloc" }
|
||||
brk_bencher = { version = "0.3.2", path = "crates/brk_bencher" }
|
||||
brk_bindgen = { version = "0.3.2", path = "crates/brk_bindgen" }
|
||||
brk_cli = { version = "0.3.2", path = "crates/brk_cli" }
|
||||
brk_client = { version = "0.3.2", path = "crates/brk_client" }
|
||||
brk_cohort = { version = "0.3.2", path = "crates/brk_cohort" }
|
||||
brk_computer = { version = "0.3.2", path = "crates/brk_computer" }
|
||||
brk_error = { version = "0.3.2", path = "crates/brk_error" }
|
||||
brk_fetcher = { version = "0.3.2", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.3.2", path = "crates/brk_indexer" }
|
||||
brk_iterator = { version = "0.3.2", path = "crates/brk_iterator" }
|
||||
brk_logger = { version = "0.3.2", path = "crates/brk_logger" }
|
||||
brk_mempool = { version = "0.3.2", path = "crates/brk_mempool" }
|
||||
brk_oracle = { version = "0.3.2", path = "crates/brk_oracle" }
|
||||
brk_query = { version = "0.3.2", path = "crates/brk_query", features = ["tokio"] }
|
||||
brk_reader = { version = "0.3.2", path = "crates/brk_reader" }
|
||||
brk_rpc = { version = "0.3.2", path = "crates/brk_rpc" }
|
||||
brk_server = { version = "0.3.2", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.3.2", path = "crates/brk_store" }
|
||||
brk_traversable = { version = "0.3.2", path = "crates/brk_traversable", features = ["pco", "derive"] }
|
||||
brk_traversable_derive = { version = "0.3.2", path = "crates/brk_traversable_derive" }
|
||||
brk_types = { version = "0.3.2", path = "crates/brk_types" }
|
||||
brk_website = { version = "0.3.2", path = "crates/brk_website" }
|
||||
byteview = "0.10.1"
|
||||
color-eyre = "0.6.5"
|
||||
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
|
||||
corepc-types = { version = "0.13.0", features = ["std"], default-features = false }
|
||||
corepc-types = { version = "0.14.0", features = ["std"], default-features = false }
|
||||
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
|
||||
fjall = "=3.0.4"
|
||||
fjall = "3.1.4"
|
||||
indexmap = { version = "2.14.0", features = ["serde"] }
|
||||
jiff = { version = "0.2.24", features = ["perf-inline", "tz-system"], default-features = false }
|
||||
jiff = { version = "0.2.28", features = ["perf-inline", "tz-system"], default-features = false }
|
||||
owo-colors = "4.3.0"
|
||||
parking_lot = "0.12.5"
|
||||
pco = "1.0.2"
|
||||
@@ -79,10 +76,10 @@ schemars = { version = "1.2.1", features = ["indexmap2"] }
|
||||
serde = "1.0.228"
|
||||
serde_bytes = "0.11.19"
|
||||
serde_derive = "1.0.228"
|
||||
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["float_roundtrip", "preserve_order"] }
|
||||
smallvec = "1.15.1"
|
||||
tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
|
||||
tower-http = { version = "0.6.10", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
|
||||
tower-http = { version = "0.6.11", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
|
||||
tower-layer = "0.3"
|
||||
tracing = { version = "0.1", default-features = false, features = ["std"] }
|
||||
ureq = { version = "3.3.0", features = ["json"] }
|
||||
@@ -95,10 +92,3 @@ tag-name = "v{{version}}"
|
||||
pre-release-commit-message = "release: v{{version}}"
|
||||
tag-message = "release: v{{version}}"
|
||||
allow-branch = ["main", "next"]
|
||||
|
||||
[workspace.metadata.dist]
|
||||
cargo-dist-version = "0.30.2"
|
||||
ci = "github"
|
||||
allow-dirty = ["ci"]
|
||||
installers = []
|
||||
targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 bitcoinresearchkit, kibo.money, satonomics
|
||||
Copyright (c) 2025 Bitcoin Research Kit
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -8,5 +8,5 @@ homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
|
||||
mimalloc = { version = "0.1.50" }
|
||||
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
|
||||
mimalloc = { version = "0.1.52" }
|
||||
|
||||
@@ -51,7 +51,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
|
||||
}
|
||||
writeln!(
|
||||
output,
|
||||
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void }}}} [options]",
|
||||
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void, cache?: boolean }}}} [options]",
|
||||
return_type
|
||||
)
|
||||
.unwrap();
|
||||
@@ -60,22 +60,22 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
|
||||
|
||||
let params = build_method_params(endpoint);
|
||||
let params_with_opts = if params.is_empty() {
|
||||
"{ signal, onValue } = {}".to_string()
|
||||
"{ signal, onValue, cache } = {}".to_string()
|
||||
} else {
|
||||
format!("{}, {{ signal, onValue }} = {{}}", params)
|
||||
format!("{}, {{ signal, onValue, cache }} = {{}}", params)
|
||||
};
|
||||
writeln!(output, " async {}({}) {{", method_name, params_with_opts).unwrap();
|
||||
|
||||
let path = build_path_template(&endpoint.path, &endpoint.path_params);
|
||||
|
||||
let fetch_call: String = if endpoint.returns_binary() {
|
||||
"this.getBytes(path, { signal, onValue })".to_string()
|
||||
"this.getBytes(path, { signal, onValue, cache })".to_string()
|
||||
} else if endpoint.returns_json() {
|
||||
"this.getJson(path, { signal, onValue })".to_string()
|
||||
"this.getJson(path, { signal, onValue, cache })".to_string()
|
||||
} else if endpoint.response_kind.text_is_numeric() {
|
||||
"Number(await this.getText(path, { signal, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
|
||||
"Number(await this.getText(path, { signal, cache, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
|
||||
} else {
|
||||
"this.getText(path, { signal, onValue })".to_string()
|
||||
"this.getText(path, { signal, onValue, cache })".to_string()
|
||||
};
|
||||
|
||||
write_path_assignment(output, endpoint, &path);
|
||||
@@ -83,7 +83,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
|
||||
if endpoint.supports_csv {
|
||||
writeln!(
|
||||
output,
|
||||
" if (format === 'csv') return this.getText(path, {{ signal, onValue }});"
|
||||
" if (format === 'csv') return this.getText(path, {{ signal, onValue, cache }});"
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -448,14 +448,17 @@ class BrkClientBase {{
|
||||
|
||||
/**
|
||||
* @param {{string}} path
|
||||
* @param {{{{ signal?: AbortSignal }}}} [options]
|
||||
* @param {{{{ signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @returns {{Promise<Response>}}
|
||||
*/
|
||||
async get(path, {{ signal }} = {{}}) {{
|
||||
async get(path, {{ signal, cache = true }} = {{}}) {{
|
||||
const url = `${{this.baseUrl}}${{path}}`;
|
||||
const signals = [AbortSignal.timeout(this.timeout)];
|
||||
if (signal) signals.push(signal);
|
||||
const res = await fetch(url, {{ signal: AbortSignal.any(signals) }});
|
||||
/** @type {{RequestInit}} */
|
||||
const init = {{ signal: AbortSignal.any(signals) }};
|
||||
if (!cache) init.cache = 'no-store';
|
||||
const res = await fetch(url, init);
|
||||
if (!res.ok) throw new BrkError(`HTTP ${{res.status}}: ${{url}}`, res.status);
|
||||
return res;
|
||||
}}
|
||||
@@ -475,14 +478,21 @@ class BrkClientBase {{
|
||||
* @template T
|
||||
* @param {{string}} path
|
||||
* @param {{(res: Response) => Promise<T>}} parse - Response body reader
|
||||
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
|
||||
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @returns {{Promise<T>}}
|
||||
*/
|
||||
async _getCached(path, parse, {{ onValue, signal }} = {{}}) {{
|
||||
async _getCached(path, parse, {{ onValue, signal, cache = true }} = {{}}) {{
|
||||
if (!cache) {{
|
||||
const res = await this.get(path, {{ signal, cache }});
|
||||
const value = await parse(res);
|
||||
if (onValue) onValue(value);
|
||||
return value;
|
||||
}}
|
||||
|
||||
const url = `${{this.baseUrl}}${{path}}`;
|
||||
/** @type {{_MemEntry<T> | undefined}} */
|
||||
const memHit = this._memGet(url);
|
||||
const browserCache = this._browserCache ?? await this._browserCachePromise;
|
||||
const browserCache = this._browserCache;
|
||||
|
||||
// L1 fast path: deliver from memCache, revalidate via network.
|
||||
// ETag match → zero parse, zero clone, zero cache write, no second onValue fire.
|
||||
@@ -497,8 +507,8 @@ class BrkClientBase {{
|
||||
this._memSet(url, netEtag, value);
|
||||
if (onValue) onValue(value);
|
||||
if (cloned && browserCache) {{
|
||||
const cache = browserCache;
|
||||
_runIdle(() => cache.put(url, cloned));
|
||||
const cacheStore = browserCache;
|
||||
_runIdle(() => cacheStore.put(url, cloned));
|
||||
}}
|
||||
return value;
|
||||
}} catch {{
|
||||
@@ -531,8 +541,8 @@ class BrkClientBase {{
|
||||
this._memSet(url, netEtag, value);
|
||||
if (onValue) onValue(value);
|
||||
if (cloned && browserCache) {{
|
||||
const cache = browserCache;
|
||||
_runIdle(() => cache.put(url, cloned));
|
||||
const cacheStore = browserCache;
|
||||
_runIdle(() => cacheStore.put(url, cloned));
|
||||
}}
|
||||
return value;
|
||||
}} catch (e) {{
|
||||
@@ -546,7 +556,7 @@ class BrkClientBase {{
|
||||
* Make a GET request expecting a JSON response. Cached and supports `onValue`.
|
||||
* @template T
|
||||
* @param {{string}} path
|
||||
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
|
||||
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @returns {{Promise<T>}}
|
||||
*/
|
||||
getJson(path, options) {{
|
||||
@@ -557,7 +567,7 @@ class BrkClientBase {{
|
||||
* Make a GET request expecting a text response (text/plain, text/csv, ...).
|
||||
* Cached and supports `onValue`, same as `getJson`.
|
||||
* @param {{string}} path
|
||||
* @param {{{{ onValue?: (value: string) => void, signal?: AbortSignal }}}} [options]
|
||||
* @param {{{{ onValue?: (value: string) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @returns {{Promise<string>}}
|
||||
*/
|
||||
getText(path, options) {{
|
||||
@@ -568,7 +578,7 @@ class BrkClientBase {{
|
||||
* Make a GET request expecting binary data (application/octet-stream).
|
||||
* Cached and supports `onValue`, same as `getJson`.
|
||||
* @param {{string}} path
|
||||
* @param {{{{ onValue?: (value: Uint8Array) => void, signal?: AbortSignal }}}} [options]
|
||||
* @param {{{{ onValue?: (value: Uint8Array) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @returns {{Promise<Uint8Array>}}
|
||||
*/
|
||||
getBytes(path, options) {{
|
||||
|
||||
@@ -31,6 +31,3 @@ vecdb = { workspace = true }
|
||||
[[bin]]
|
||||
name = "brk"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.dist]
|
||||
dist = true
|
||||
|
||||
@@ -30,6 +30,10 @@ Portable build (without native CPU optimizations):
|
||||
cargo install --locked brk_cli
|
||||
```
|
||||
|
||||
## Update
|
||||
|
||||
Updating is the same as installing: re-run the install command above. `cargo install` overwrites the existing binary in place. Indexed data is reused when the on-disk format is unchanged, otherwise it is reset and re-synced automatically on the next run.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
|
||||
@@ -3534,7 +3534,7 @@ pub struct SeriesTree {
|
||||
pub investing: SeriesTree_Investing,
|
||||
pub market: SeriesTree_Market,
|
||||
pub pools: SeriesTree_Pools,
|
||||
pub prices: SeriesTree_Prices,
|
||||
pub price: SeriesTree_Price,
|
||||
pub supply: SeriesTree_Supply,
|
||||
pub cohorts: SeriesTree_Cohorts,
|
||||
}
|
||||
@@ -3556,7 +3556,7 @@ impl SeriesTree {
|
||||
investing: SeriesTree_Investing::new(client.clone(), format!("{base_path}_investing")),
|
||||
market: SeriesTree_Market::new(client.clone(), format!("{base_path}_market")),
|
||||
pools: SeriesTree_Pools::new(client.clone(), format!("{base_path}_pools")),
|
||||
prices: SeriesTree_Prices::new(client.clone(), format!("{base_path}_prices")),
|
||||
price: SeriesTree_Price::new(client.clone(), format!("{base_path}_price")),
|
||||
supply: SeriesTree_Supply::new(client.clone(), format!("{base_path}_supply")),
|
||||
cohorts: SeriesTree_Cohorts::new(client.clone(), format!("{base_path}_cohorts")),
|
||||
}
|
||||
@@ -7063,31 +7063,31 @@ impl SeriesTree_Pools_Minor {
|
||||
}
|
||||
|
||||
/// Series tree node.
|
||||
pub struct SeriesTree_Prices {
|
||||
pub split: SeriesTree_Prices_Split,
|
||||
pub ohlc: SeriesTree_Prices_Ohlc,
|
||||
pub spot: SeriesTree_Prices_Spot,
|
||||
pub struct SeriesTree_Price {
|
||||
pub split: SeriesTree_Price_Split,
|
||||
pub ohlc: SeriesTree_Price_Ohlc,
|
||||
pub spot: SeriesTree_Price_Spot,
|
||||
}
|
||||
|
||||
impl SeriesTree_Prices {
|
||||
impl SeriesTree_Price {
|
||||
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
|
||||
Self {
|
||||
split: SeriesTree_Prices_Split::new(client.clone(), format!("{base_path}_split")),
|
||||
ohlc: SeriesTree_Prices_Ohlc::new(client.clone(), format!("{base_path}_ohlc")),
|
||||
spot: SeriesTree_Prices_Spot::new(client.clone(), format!("{base_path}_spot")),
|
||||
split: SeriesTree_Price_Split::new(client.clone(), format!("{base_path}_split")),
|
||||
ohlc: SeriesTree_Price_Ohlc::new(client.clone(), format!("{base_path}_ohlc")),
|
||||
spot: SeriesTree_Price_Spot::new(client.clone(), format!("{base_path}_spot")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Series tree node.
|
||||
pub struct SeriesTree_Prices_Split {
|
||||
pub struct SeriesTree_Price_Split {
|
||||
pub open: CentsSatsUsdPattern3,
|
||||
pub high: CentsSatsUsdPattern3,
|
||||
pub low: CentsSatsUsdPattern3,
|
||||
pub close: CentsSatsUsdPattern3,
|
||||
}
|
||||
|
||||
impl SeriesTree_Prices_Split {
|
||||
impl SeriesTree_Price_Split {
|
||||
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
|
||||
Self {
|
||||
open: CentsSatsUsdPattern3::new(client.clone(), "price_open".to_string()),
|
||||
@@ -7099,13 +7099,13 @@ impl SeriesTree_Prices_Split {
|
||||
}
|
||||
|
||||
/// Series tree node.
|
||||
pub struct SeriesTree_Prices_Ohlc {
|
||||
pub struct SeriesTree_Price_Ohlc {
|
||||
pub usd: SeriesPattern2<OHLCDollars>,
|
||||
pub cents: SeriesPattern2<OHLCCents>,
|
||||
pub sats: SeriesPattern2<OHLCSats>,
|
||||
}
|
||||
|
||||
impl SeriesTree_Prices_Ohlc {
|
||||
impl SeriesTree_Price_Ohlc {
|
||||
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
|
||||
Self {
|
||||
usd: SeriesPattern2::new(client.clone(), "price_ohlc".to_string()),
|
||||
@@ -7116,13 +7116,13 @@ impl SeriesTree_Prices_Ohlc {
|
||||
}
|
||||
|
||||
/// Series tree node.
|
||||
pub struct SeriesTree_Prices_Spot {
|
||||
pub struct SeriesTree_Price_Spot {
|
||||
pub usd: SeriesPattern1<Dollars>,
|
||||
pub cents: SeriesPattern1<Cents>,
|
||||
pub sats: SeriesPattern1<Sats>,
|
||||
}
|
||||
|
||||
impl SeriesTree_Prices_Spot {
|
||||
impl SeriesTree_Price_Spot {
|
||||
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
|
||||
Self {
|
||||
usd: SeriesPattern1::new(client.clone(), "price".to_string()),
|
||||
@@ -8953,7 +8953,7 @@ pub struct BrkClient {
|
||||
|
||||
impl BrkClient {
|
||||
/// Client version.
|
||||
pub const VERSION: &'static str = "v0.3.0-beta.9";
|
||||
pub const VERSION: &'static str = "v0.3.1";
|
||||
|
||||
/// Create a new client with the given base URL.
|
||||
pub fn new(base_url: impl Into<String>) -> Self {
|
||||
@@ -9856,6 +9856,51 @@ impl BrkClient {
|
||||
self.base.get_json(&format!("/api/mempool/price"))
|
||||
}
|
||||
|
||||
/// Live BTC/USD price
|
||||
///
|
||||
/// Current BTC/USD price in dollars. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/price`
|
||||
pub fn get_oracle_price(&self) -> Result<Dollars> {
|
||||
self.base.get_json(&format!("/api/oracle/price"))
|
||||
}
|
||||
|
||||
/// Live payment output histogram
|
||||
///
|
||||
/// Live smoothed histogram of oracle-eligible payment outputs, binned by output value on the oracle log scale. It combines the committed oracle window with the forming mempool block. A flat array of log-scale bins.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/payments/live`
|
||||
pub fn get_oracle_histogram_payments_live(&self) -> Result<Vec<i64>> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/payments/live"))
|
||||
}
|
||||
|
||||
/// Payment output histogram at height or day
|
||||
///
|
||||
/// Smoothed histogram of oracle-eligible payment outputs for a confirmed point. A block height (`840000`) gives that block's oracle payment histogram; a calendar date (`YYYY-MM-DD`) gives the average of that day's per-block payment histograms. A flat array of log-scale bins.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/payments/{point}`
|
||||
pub fn get_oracle_histogram_payments(&self, point: &str) -> Result<Vec<i64>> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/payments/{point}"))
|
||||
}
|
||||
|
||||
/// Live output value histogram
|
||||
///
|
||||
/// Live unfiltered output value histogram for the forming mempool block. Every live output is binned by value on the oracle log scale; no oracle payment filters are applied. A flat array of log-scale bins, all zero when no mempool is configured.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/outputs/live`
|
||||
pub fn get_oracle_histogram_outputs_live(&self) -> Result<Vec<i64>> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/outputs/live"))
|
||||
}
|
||||
|
||||
/// Output value histogram at height or day
|
||||
///
|
||||
/// Unfiltered output value histogram for a confirmed point. A block height (`840000`) gives every output in that block, coinbase included, binned by value on the oracle log scale; a calendar date (`YYYY-MM-DD`) sums every block that day. A flat array of log-scale bins.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/outputs/{point}`
|
||||
pub fn get_oracle_histogram_outputs(&self, point: &str) -> Result<Vec<i64>> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/outputs/{point}"))
|
||||
}
|
||||
|
||||
/// Txid by index
|
||||
///
|
||||
/// Retrieve the transaction ID (txid) at a given global transaction index. Returns the txid as plain text.
|
||||
|
||||
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, distribution, mining, prices, supply};
|
||||
use crate::{blocks, distribution, mining, price, supply};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
mining: &mining::Vecs,
|
||||
supply_vecs: &supply::Vecs,
|
||||
|
||||
@@ -5,14 +5,14 @@ use vecdb::Exit;
|
||||
|
||||
use super::super::{activity, cap, supply};
|
||||
use super::Vecs;
|
||||
use crate::{distribution, prices};
|
||||
use crate::{distribution, price};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
distribution: &distribution::Vecs,
|
||||
activity: &activity::Vecs,
|
||||
supply: &supply::Vecs,
|
||||
|
||||
@@ -4,14 +4,14 @@ use brk_types::StoredF64;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::{super::value, Vecs};
|
||||
use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, prices};
|
||||
use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
value: &value::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -4,13 +4,13 @@ use vecdb::Exit;
|
||||
|
||||
use super::super::activity;
|
||||
use super::Vecs;
|
||||
use crate::{distribution, prices};
|
||||
use crate::{distribution, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
distribution: &distribution::Vecs,
|
||||
activity: &activity::Vecs,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -5,13 +5,13 @@ use vecdb::Exit;
|
||||
|
||||
use super::super::activity;
|
||||
use super::Vecs;
|
||||
use crate::{distribution, prices};
|
||||
use crate::{distribution, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
distribution: &distribution::Vecs,
|
||||
activity: &activity::Vecs,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -45,7 +45,7 @@ use super::{
|
||||
count::AddrCountFundedTotalVecs,
|
||||
supply::{AddrSupplyShareVecs, AddrSupplyVecs},
|
||||
};
|
||||
use crate::{indexes, prices};
|
||||
use crate::{indexes, price};
|
||||
|
||||
mod state;
|
||||
|
||||
@@ -104,7 +104,7 @@ impl ExposedAddrVecs {
|
||||
pub(crate) fn compute_rest(
|
||||
&mut self,
|
||||
starting_lengths: &Lengths,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -35,7 +35,7 @@ use super::{
|
||||
use crate::{
|
||||
indexes, inputs,
|
||||
internal::{WindowStartVec, Windows},
|
||||
outputs, prices,
|
||||
outputs, price,
|
||||
};
|
||||
|
||||
mod state;
|
||||
@@ -112,7 +112,7 @@ impl ReusedAddrVecs {
|
||||
starting_lengths: &Lengths,
|
||||
outputs_by_type: &outputs::ByTypeVecs,
|
||||
inputs_by_type: &inputs::ByTypeVecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::{
|
||||
distribution::DynCohortVecs,
|
||||
indexes,
|
||||
internal::{WindowStartVec, Windows},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use super::{super::traits::CohortVecs, vecs::AddrCohortVecs};
|
||||
@@ -95,7 +95,7 @@ impl AddrCohorts {
|
||||
/// First phase of post-processing: compute index transforms.
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -108,7 +108,7 @@ impl AddrCohorts {
|
||||
/// Second phase of post-processing: compute relative metrics.
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::{
|
||||
distribution::state::{AddrCohortState, MinimalRealizedState},
|
||||
indexes,
|
||||
internal::{PerBlockWithDeltas, WindowStartVec, Windows},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use crate::distribution::metrics::{ImportConfig, MinimalCohortMetrics};
|
||||
@@ -174,7 +174,7 @@ impl DynCohortVecs for AddrCohortVecs {
|
||||
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -229,7 +229,7 @@ impl CohortVecs for AddrCohortVecs {
|
||||
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -3,7 +3,7 @@ use brk_indexer::Lengths;
|
||||
use brk_types::{Cents, Height, Sats, StoredU64, Version};
|
||||
use vecdb::{Exit, ReadableVec};
|
||||
|
||||
use crate::prices;
|
||||
use crate::price;
|
||||
|
||||
/// Dynamic dispatch trait for cohort vectors.
|
||||
///
|
||||
@@ -31,7 +31,7 @@ pub trait DynCohortVecs: Send + Sync {
|
||||
/// First phase of post-processing computations.
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
@@ -61,7 +61,7 @@ pub trait CohortVecs: DynCohortVecs {
|
||||
/// Second phase of post-processing computations.
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::{
|
||||
},
|
||||
indexes,
|
||||
internal::{ValuePerBlockCumulativeRolling, WindowStartVec, Windows},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use super::{fenwick::CostBasisFenwick, vecs::UTXOCohortVecs};
|
||||
@@ -483,7 +483,7 @@ impl UTXOCohorts<Rw> {
|
||||
/// First phase of post-processing: compute index transforms.
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -546,7 +546,7 @@ impl UTXOCohorts<Rw> {
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -6,7 +6,7 @@ use vecdb::{Exit, ReadableVec};
|
||||
|
||||
use crate::{
|
||||
distribution::{cohorts::traits::DynCohortVecs, metrics::CoreCohortMetrics},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use super::UTXOCohortVecs;
|
||||
@@ -56,7 +56,7 @@ impl DynCohortVecs for UTXOCohortVecs<CoreCohortMetrics> {
|
||||
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -6,7 +6,7 @@ use vecdb::{Exit, ReadableVec};
|
||||
|
||||
use crate::{
|
||||
distribution::{cohorts::traits::DynCohortVecs, metrics::MinimalCohortMetrics},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use super::UTXOCohortVecs;
|
||||
@@ -49,7 +49,7 @@ impl DynCohortVecs for UTXOCohortVecs<MinimalCohortMetrics> {
|
||||
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -55,7 +55,7 @@ use crate::{
|
||||
metrics::{CohortMetricsBase, CohortMetricsState},
|
||||
state::UTXOCohortState,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
#[derive(Traversable)]
|
||||
@@ -186,7 +186,7 @@ impl<M: CohortMetricsBase + Traversable> DynCohortVecs for UTXOCohortVecs<M> {
|
||||
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -5,7 +5,7 @@ use brk_types::{Cents, Height, Version};
|
||||
use vecdb::{Exit, ReadableVec};
|
||||
|
||||
use crate::{
|
||||
distribution::cohorts::traits::DynCohortVecs, distribution::metrics::TypeCohortMetrics, prices,
|
||||
distribution::cohorts::traits::DynCohortVecs, distribution::metrics::TypeCohortMetrics, price,
|
||||
};
|
||||
|
||||
use super::UTXOCohortVecs;
|
||||
@@ -55,7 +55,7 @@ impl DynCohortVecs for UTXOCohortVecs<TypeCohortMetrics> {
|
||||
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -24,51 +24,60 @@ pub struct RecoveredState {
|
||||
/// Returns Height::ZERO if any validation fails (triggers fresh start).
|
||||
pub(crate) fn recover_state(
|
||||
height: Height,
|
||||
chain_state_rollback: vecdb::Result<Stamp>,
|
||||
chain_state_rollback: Option<vecdb::Result<Stamp>>,
|
||||
any_addr_indexes: &mut AnyAddrIndexesVecs,
|
||||
addrs_data: &mut AddrsDataVecs,
|
||||
utxo_cohorts: &mut UTXOCohorts,
|
||||
addr_cohorts: &mut AddrCohorts,
|
||||
) -> Result<RecoveredState> {
|
||||
let stamp = Stamp::from(height);
|
||||
// `None`: clean resume, already at the checkpoint, nothing to undo.
|
||||
// `Some`: reorg, undo state past the resume point.
|
||||
let consistent_height = match chain_state_rollback {
|
||||
None => height,
|
||||
Some(chain_state_rollback) => {
|
||||
let stamp = Stamp::from(height);
|
||||
|
||||
// Rollback address state vectors
|
||||
let addr_indexes_rollback = any_addr_indexes.rollback_before(stamp);
|
||||
let addr_data_rollback = addrs_data.rollback_before(stamp);
|
||||
// Rollback address state vectors
|
||||
let addr_indexes_rollback = any_addr_indexes.rollback_before(stamp);
|
||||
let addr_data_rollback = addrs_data.rollback_before(stamp);
|
||||
|
||||
// Verify rollback consistency - all must agree on the same height
|
||||
let consistent_height = rollback_states(
|
||||
chain_state_rollback,
|
||||
addr_indexes_rollback,
|
||||
addr_data_rollback,
|
||||
);
|
||||
// Verify rollback consistency - all must agree on the same height
|
||||
let consistent_height = rollback_states(
|
||||
chain_state_rollback,
|
||||
addr_indexes_rollback,
|
||||
addr_data_rollback,
|
||||
);
|
||||
|
||||
// If rollbacks are inconsistent, start fresh
|
||||
if consistent_height.is_zero() {
|
||||
warn!("Rollback consistency check failed: inconsistent heights");
|
||||
return Ok(RecoveredState {
|
||||
starting_height: Height::ZERO,
|
||||
});
|
||||
}
|
||||
// If rollbacks are inconsistent, start fresh
|
||||
if consistent_height.is_zero() {
|
||||
warn!("Rollback consistency check failed: inconsistent heights");
|
||||
return Ok(RecoveredState {
|
||||
starting_height: Height::ZERO,
|
||||
});
|
||||
}
|
||||
|
||||
// Rollback can land at an earlier height (multi-block change file), which is fine.
|
||||
// But if it lands AHEAD of target, that means rollback failed (missing change files).
|
||||
if consistent_height > height {
|
||||
warn!(
|
||||
"Rollback failed: still at {} but target was {}, falling back to fresh start",
|
||||
consistent_height, height
|
||||
);
|
||||
return Ok(RecoveredState {
|
||||
starting_height: Height::ZERO,
|
||||
});
|
||||
}
|
||||
// Rollback can land at an earlier height (multi-block change file), which is fine.
|
||||
// But if it lands AHEAD of target, that means rollback failed (missing change files).
|
||||
if consistent_height > height {
|
||||
warn!(
|
||||
"Rollback failed: still at {} but target was {}, falling back to fresh start",
|
||||
consistent_height, height
|
||||
);
|
||||
return Ok(RecoveredState {
|
||||
starting_height: Height::ZERO,
|
||||
});
|
||||
}
|
||||
|
||||
if consistent_height != height {
|
||||
debug!(
|
||||
"Rollback landed at {} instead of {}, will resume from there",
|
||||
consistent_height, height
|
||||
);
|
||||
}
|
||||
if consistent_height != height {
|
||||
debug!(
|
||||
"Rollback landed at {} instead of {}, will resume from there",
|
||||
consistent_height, height
|
||||
);
|
||||
}
|
||||
|
||||
consistent_height
|
||||
}
|
||||
};
|
||||
|
||||
// Import UTXO cohort states - all must succeed
|
||||
debug!(
|
||||
|
||||
@@ -11,7 +11,7 @@ use crate::{
|
||||
state::{CohortState, CostBasisOps, RealizedOps},
|
||||
},
|
||||
internal::{PerBlockCumulativeRolling, ValuePerBlockCumulativeRolling},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use super::ActivityMinimal;
|
||||
@@ -98,7 +98,7 @@ impl ActivityCore {
|
||||
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::{
|
||||
metrics::ImportConfig,
|
||||
state::{CohortState, CostBasisOps, RealizedOps},
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use super::ActivityCore;
|
||||
@@ -89,7 +89,7 @@ impl ActivityFull {
|
||||
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
state::{CohortState, CostBasisOps, RealizedOps},
|
||||
},
|
||||
internal::ValuePerBlockCumulativeRolling,
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
#[derive(Traversable)]
|
||||
@@ -63,7 +63,7 @@ impl ActivityMinimal {
|
||||
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -13,7 +13,7 @@ use vecdb::Exit;
|
||||
|
||||
use crate::{
|
||||
distribution::state::{CohortState, CostBasisOps, RealizedOps},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
pub trait ActivityLike: Send + Sync {
|
||||
@@ -30,7 +30,7 @@ pub trait ActivityLike: Send + Sync {
|
||||
) -> Result<()>;
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
@@ -62,7 +62,7 @@ impl ActivityLike for ActivityCore {
|
||||
}
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -96,7 +96,7 @@ impl ActivityLike for ActivityFull {
|
||||
}
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -11,7 +11,7 @@ use crate::{
|
||||
ActivityFull, AdjustedSopr, CohortMetricsBase, CostBasis, ImportConfig, OutputsBase,
|
||||
RealizedFull, RelativeForAll, SupplyCore, UnrealizedFull,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
/// All-cohort metrics: extended realized + adjusted (as composable add-on),
|
||||
@@ -100,7 +100,7 @@ impl AllCohortMetrics {
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
|
||||
under_1h_value_created: &impl ReadableVec<Height, Cents>,
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
ActivityCore, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore, SupplyCore,
|
||||
UnrealizedCore,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
/// Basic cohort metrics: no extensions, used by age_range cohorts.
|
||||
@@ -61,7 +61,7 @@ impl BasicCohortMetrics {
|
||||
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
ActivityCore, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore, SupplyCore,
|
||||
UnrealizedCore,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
#[derive(Traversable)]
|
||||
@@ -102,7 +102,7 @@ impl CoreCohortMetrics {
|
||||
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -122,7 +122,7 @@ impl CoreCohortMetrics {
|
||||
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::{
|
||||
ActivityFull, CohortMetricsBase, CostBasis, ImportConfig, OutputsBase, RealizedFull,
|
||||
RelativeWithExtended, SupplyCore, UnrealizedFull,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
/// Cohort metrics with extended realized + extended cost basis (no adjusted).
|
||||
@@ -90,7 +90,7 @@ impl ExtendedCohortMetrics {
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
distribution::metrics::{
|
||||
ActivityFull, AdjustedSopr, CohortMetricsBase, ImportConfig, RealizedFull, UnrealizedFull,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use super::ExtendedCohortMetrics;
|
||||
@@ -62,7 +62,7 @@ impl ExtendedAdjustedCohortMetrics {
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
|
||||
under_1h_value_created: &impl ReadableVec<Height, Cents>,
|
||||
|
||||
@@ -9,7 +9,7 @@ use crate::{
|
||||
distribution::metrics::{
|
||||
ActivityMinimal, ImportConfig, OutputsBase, RealizedMinimal, SupplyBase, UnrealizedMinimal,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
/// MinimalCohortMetrics: supply, outputs, realized cap/price/mvrv/profit/loss + value_created/destroyed.
|
||||
@@ -97,7 +97,7 @@ impl MinimalCohortMetrics {
|
||||
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -111,7 +111,7 @@ impl MinimalCohortMetrics {
|
||||
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -9,7 +9,7 @@ use crate::{
|
||||
distribution::metrics::{
|
||||
ActivityMinimal, ImportConfig, OutputsBase, RealizedMinimal, SupplyCore, UnrealizedBasic,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
/// TypeCohortMetrics: supply(core), outputs(base), realized(minimal), unrealized(basic).
|
||||
@@ -59,7 +59,7 @@ impl TypeCohortMetrics {
|
||||
|
||||
pub(crate) fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -73,7 +73,7 @@ impl TypeCohortMetrics {
|
||||
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
all_supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -149,7 +149,7 @@ use crate::{
|
||||
CohortState, CoreRealizedState, CostBasisData, CostBasisOps, CostBasisRaw,
|
||||
MinimalRealizedState, RealizedOps, RealizedState, WithCapital, WithoutCapital,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
pub trait CohortMetricsState {
|
||||
@@ -270,7 +270,7 @@ pub trait CohortMetricsBase:
|
||||
/// First phase of computed metrics (indexes from height).
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
internal::{
|
||||
PerBlock, RatioPerBlock, ValuePerBlock, ValuePerBlockWithDeltas, WindowStartVec, Windows,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
#[derive(Traversable)]
|
||||
@@ -115,7 +115,7 @@ impl ProfitabilityBucket {
|
||||
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
is_profit: bool,
|
||||
exit: &Exit,
|
||||
@@ -176,7 +176,7 @@ impl ProfitabilityBucket {
|
||||
|
||||
pub(crate) fn compute_from_ranges(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
is_profit: bool,
|
||||
sources: &[&ProfitabilityBucket],
|
||||
@@ -293,7 +293,7 @@ impl ProfitabilityMetrics {
|
||||
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::{
|
||||
FiatPerBlockCumulativeWithSumsAndDeltas, LazyPerBlock, NegCentsUnsignedToDollars,
|
||||
PerBlockCumulativeRolling, RatioCents64, RollingWindow24hPerBlock, Windows,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use crate::distribution::metrics::ImportConfig;
|
||||
@@ -166,7 +166,7 @@ impl RealizedCore {
|
||||
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
|
||||
transfer_volume_sum_24h_cents: &impl ReadableVec<Height, Cents>,
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::{
|
||||
RatioPerBlockStdDevBands, RatioSma, RollingWindows, RollingWindowsFrom1w,
|
||||
ValuePerBlockCumulativeRolling,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use crate::distribution::metrics::ImportConfig;
|
||||
@@ -240,7 +240,7 @@ impl RealizedFull {
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
|
||||
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::{
|
||||
FiatPerBlockCumulativeWithSums, FiatPerBlockWithDeltas, Identity, LazyPerBlock,
|
||||
PriceWithRatioPerBlock,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use crate::distribution::metrics::ImportConfig;
|
||||
@@ -104,7 +104,7 @@ impl RealizedMinimal {
|
||||
|
||||
pub(crate) fn compute_rest_part2(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -3,7 +3,7 @@ use brk_traversable::Traversable;
|
||||
use brk_types::{Height, Sats, StoredU64, Version};
|
||||
use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode, WritableVec};
|
||||
|
||||
use crate::{indexes, internal::ValuePerBlock, prices};
|
||||
use crate::{indexes, internal::ValuePerBlock, price};
|
||||
|
||||
/// Average amount held per UTXO and per funded address.
|
||||
///
|
||||
@@ -53,7 +53,7 @@ impl AvgAmountMetrics {
|
||||
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
supply_sats: &impl ReadableVec<Height, Sats>,
|
||||
utxo_count: &impl ReadableVec<Height, StoredU64>,
|
||||
funded_addr_count: &impl ReadableVec<Height, StoredU64>,
|
||||
|
||||
@@ -6,7 +6,7 @@ use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVe
|
||||
|
||||
use crate::{
|
||||
distribution::state::{CohortState, CostBasisOps, RealizedOps},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
use crate::internal::{
|
||||
@@ -64,7 +64,7 @@ impl SupplyBase {
|
||||
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
max_from: Height,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -5,7 +5,7 @@ use brk_types::{Height, Version};
|
||||
use derive_more::{Deref, DerefMut};
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode, WritableVec};
|
||||
|
||||
use crate::{distribution::state::UnrealizedState, prices};
|
||||
use crate::{distribution::state::UnrealizedState, price};
|
||||
|
||||
use crate::internal::{
|
||||
HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin, LazyValuePerBlock, ValuePerBlock,
|
||||
@@ -72,7 +72,7 @@ impl SupplyCore {
|
||||
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
max_from: Height,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -7,7 +7,7 @@ use vecdb::{AnyStoredVec, AnyVec, BytesVec, Exit, ReadableVec, Rw, StorageMode,
|
||||
|
||||
use crate::distribution::state::UnrealizedState;
|
||||
use crate::internal::{CentsSubtractToCentsSigned, FiatPerBlock};
|
||||
use crate::{distribution::metrics::ImportConfig, prices};
|
||||
use crate::{distribution::metrics::ImportConfig, price};
|
||||
|
||||
use super::UnrealizedCore;
|
||||
|
||||
@@ -99,7 +99,7 @@ impl UnrealizedFull {
|
||||
|
||||
pub(crate) fn compute_rest_all(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
|
||||
@@ -13,7 +13,7 @@ use brk_indexer::Lengths;
|
||||
use brk_types::{Height, Sats};
|
||||
use vecdb::{Exit, ReadableVec};
|
||||
|
||||
use crate::{distribution::state::UnrealizedState, prices};
|
||||
use crate::{distribution::state::UnrealizedState, price};
|
||||
|
||||
pub trait UnrealizedLike: Send + Sync {
|
||||
fn as_core(&self) -> &UnrealizedCore;
|
||||
@@ -22,7 +22,7 @@ pub trait UnrealizedLike: Send + Sync {
|
||||
fn push_state(&mut self, state: &UnrealizedState);
|
||||
fn compute_rest(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
@@ -46,7 +46,7 @@ impl UnrealizedLike for UnrealizedCore {
|
||||
}
|
||||
fn compute_rest(
|
||||
&mut self,
|
||||
_prices: &prices::Vecs,
|
||||
_prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
_supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
_supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
@@ -72,7 +72,7 @@ impl UnrealizedLike for UnrealizedFull {
|
||||
}
|
||||
fn compute_rest(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::{
|
||||
PerBlockCumulativeRolling, WindowStartVec, Windows, WithAddrTypes,
|
||||
db_utils::{finalize_db, open_db},
|
||||
},
|
||||
outputs, prices, transactions,
|
||||
outputs, price, transactions,
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -316,7 +316,7 @@ impl Vecs {
|
||||
outputs: &outputs::Vecs,
|
||||
transactions: &transactions::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.db.sync_bg_tasks()?;
|
||||
@@ -341,12 +341,13 @@ impl Vecs {
|
||||
// Try to resume from checkpoint, fall back to fresh start if needed
|
||||
let recovered_height = match start_mode {
|
||||
StartMode::Resume(height) => {
|
||||
let stamp = Stamp::from(height);
|
||||
// Roll back only on a reorg. A clean resume has nothing to undo, and an
|
||||
// interrupted run wrote no rollback metadata (periodic flushes use
|
||||
// with_changes=false; only the final write creates the `changes/` dir),
|
||||
// so `rollback_before` would fail with `NotFound`.
|
||||
let chain_state_rollback = (height < current_height)
|
||||
.then(|| self.supply_state.rollback_before(Stamp::from(height)));
|
||||
|
||||
// Rollback BytesVec state and capture results for validation
|
||||
let chain_state_rollback = self.supply_state.rollback_before(stamp);
|
||||
|
||||
// Validate all rollbacks and imports are consistent
|
||||
let recovered = recover_state(
|
||||
height,
|
||||
chain_state_rollback,
|
||||
|
||||
@@ -6,7 +6,7 @@ use brk_traversable::Traversable;
|
||||
use brk_types::Version;
|
||||
use vecdb::{Database, Exit, Rw, StorageMode};
|
||||
|
||||
use crate::{distribution, indexes, prices};
|
||||
use crate::{distribution, indexes, price};
|
||||
|
||||
pub use inner::RarityMeterInner;
|
||||
|
||||
@@ -37,7 +37,7 @@ impl RarityMeter {
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
distribution: &distribution::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let realized = &distribution.utxo_cohorts.all.metrics.realized;
|
||||
|
||||
@@ -6,7 +6,7 @@ use derive_more::{Deref, DerefMut};
|
||||
use vecdb::{Database, EagerVec, Exit, PcoVec, ReadableVec, Rw, StorageMode};
|
||||
|
||||
use crate::internal::{LazyPerBlock, PerBlock, Price};
|
||||
use crate::{indexes, prices};
|
||||
use crate::{indexes, price};
|
||||
|
||||
use super::{RatioPerBlock, RatioPerBlockPercentiles};
|
||||
|
||||
@@ -63,7 +63,7 @@ impl PriceWithRatioPerBlock {
|
||||
/// Compute price via closure (in cents), then compute ratio.
|
||||
pub(crate) fn compute_all<F>(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
mut compute_price: F,
|
||||
@@ -101,7 +101,7 @@ impl PriceWithRatioExtendedPerBlock {
|
||||
/// Compute ratio and percentiles from already-computed price cents.
|
||||
pub(crate) fn compute_rest(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -120,7 +120,7 @@ impl PriceWithRatioExtendedPerBlock {
|
||||
/// Compute price via closure (in cents), then compute ratio and percentiles.
|
||||
pub(crate) fn compute_all<F>(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
mut compute_price: F,
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
CentsUnsignedToDollars, LazyPerBlock, NumericValue, PerBlock, SatsSignedToBitcoin,
|
||||
SatsToBitcoin, SatsToCents,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
/// Trait that associates a sats type with its transform to Bitcoin.
|
||||
@@ -69,7 +69,7 @@ impl ValuePerBlock {
|
||||
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
max_from: Height,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -8,7 +8,7 @@ use vecdb::{
|
||||
|
||||
use crate::{
|
||||
internal::{CentsUnsignedToDollars, SatsToBitcoin, SatsToCents},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
/// Raw per-block amount data: sats + cents (stored), btc + usd (lazy), no resolutions.
|
||||
@@ -44,7 +44,7 @@ impl ValueBlock {
|
||||
pub(crate) fn compute_cents(
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.cents.compute_binary::<Sats, Cents, SatsToCents>(
|
||||
|
||||
@@ -6,7 +6,7 @@ use vecdb::{Database, EagerVec, Exit, PcoVec, Rw, StorageMode};
|
||||
use crate::{
|
||||
indexes,
|
||||
internal::{ValueBlock, ValuePerBlock},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
#[derive(Traversable)]
|
||||
@@ -39,7 +39,7 @@ impl ValuePerBlockCumulative {
|
||||
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
max_from: Height,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -61,7 +61,7 @@ impl ValuePerBlockCumulative {
|
||||
pub(crate) fn compute_with(
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
LazyRollingAvgsAmountFromHeight, LazyRollingSumsAmountFromHeight, ValuePerBlockCumulative,
|
||||
WindowStartVec, Windows,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
#[derive(Deref, DerefMut, Traversable)]
|
||||
@@ -63,7 +63,7 @@ impl ValuePerBlockCumulativeRolling {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
|
||||
) -> Result<()> {
|
||||
@@ -74,7 +74,7 @@ impl ValuePerBlockCumulativeRolling {
|
||||
pub(crate) fn compute_rest(
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.inner.compute(prices, max_from, exit)
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
RollingDistributionValuePerBlock, ValuePerBlockCumulativeRolling, WindowStartVec,
|
||||
WindowStarts, Windows,
|
||||
},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
#[derive(Deref, DerefMut, Traversable)]
|
||||
@@ -49,7 +49,7 @@ impl ValuePerBlockFull {
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
windows: &WindowStarts<'_>,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -11,7 +11,7 @@ use rayon::prelude::*;
|
||||
use schemars::JsonSchema;
|
||||
use vecdb::{AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, WritableVec};
|
||||
|
||||
use crate::{indexes, prices};
|
||||
use crate::{indexes, price};
|
||||
|
||||
use super::{
|
||||
BpsType, NumericValue, PerBlock, PerBlockCumulativeRolling, PercentPerBlock, ValuePerBlock,
|
||||
@@ -229,7 +229,7 @@ impl WithAddrTypes<ValuePerBlock> {
|
||||
pub(crate) fn compute_rest(
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.all.compute(prices, max_from, exit)?;
|
||||
|
||||
@@ -4,7 +4,7 @@ use brk_types::{BasisPointsSigned32, Bitcoin, Cents, Date, Day1, Dollars, Sats};
|
||||
use vecdb::{AnyVec, Exit, ReadableOptionVec, ReadableVec, VecIndex};
|
||||
|
||||
use super::{ByDcaPeriod, Vecs};
|
||||
use crate::{blocks, indexes, internal::RatioDiffCentsBps32, market, prices};
|
||||
use crate::{blocks, indexes, internal::RatioDiffCentsBps32, market, price};
|
||||
|
||||
const DCA_AMOUNT: Dollars = Dollars::mint(100.0);
|
||||
|
||||
@@ -13,7 +13,7 @@ impl Vecs {
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
lookback: &market::lookback::Vecs,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -22,7 +22,7 @@ mod market;
|
||||
mod mining;
|
||||
mod outputs;
|
||||
mod pools;
|
||||
pub mod prices;
|
||||
pub mod price;
|
||||
mod supply;
|
||||
mod transactions;
|
||||
|
||||
@@ -38,7 +38,7 @@ pub struct Computer<M: StorageMode = Rw> {
|
||||
pub investing: Box<investing::Vecs<M>>,
|
||||
pub market: Box<market::Vecs<M>>,
|
||||
pub pools: Box<pools::Vecs<M>>,
|
||||
pub prices: Box<prices::Vecs<M>>,
|
||||
pub price: Box<price::Vecs<M>>,
|
||||
#[traversable(flatten)]
|
||||
pub distribution: Box<distribution::Vecs<M>>,
|
||||
pub supply: Box<supply::Vecs<M>>,
|
||||
@@ -66,14 +66,14 @@ impl Computer {
|
||||
)?))
|
||||
})?;
|
||||
|
||||
let (constants, prices) = timed("Imported prices/constants", || -> Result<_> {
|
||||
let (constants, price) = timed("Imported price/constants", || -> Result<_> {
|
||||
let constants = Box::new(constants::Vecs::new(VERSION, &indexes));
|
||||
let prices = Box::new(prices::Vecs::forced_import(
|
||||
let price = Box::new(price::Vecs::forced_import(
|
||||
&computed_path,
|
||||
VERSION,
|
||||
&indexes,
|
||||
)?);
|
||||
Ok((constants, prices))
|
||||
Ok((constants, price))
|
||||
})?;
|
||||
|
||||
let blocks = timed("Imported blocks", || -> Result<_> {
|
||||
@@ -223,7 +223,7 @@ impl Computer {
|
||||
cointime,
|
||||
indexes,
|
||||
inputs,
|
||||
prices,
|
||||
price,
|
||||
outputs,
|
||||
};
|
||||
|
||||
@@ -244,7 +244,7 @@ impl Computer {
|
||||
investing::DB_NAME,
|
||||
market::DB_NAME,
|
||||
pools::DB_NAME,
|
||||
prices::DB_NAME,
|
||||
price::DB_NAME,
|
||||
distribution::DB_NAME,
|
||||
supply::DB_NAME,
|
||||
inputs::DB_NAME,
|
||||
@@ -297,8 +297,8 @@ impl Computer {
|
||||
})
|
||||
},
|
||||
|| {
|
||||
timed("Computed prices", || {
|
||||
self.prices.compute(indexer, &self.indexes, exit)
|
||||
timed("Computed price", || {
|
||||
self.price.compute(indexer, &self.indexes, exit)
|
||||
})
|
||||
},
|
||||
);
|
||||
@@ -310,7 +310,7 @@ impl Computer {
|
||||
let market = scope.spawn(|| {
|
||||
timed("Computed market", || {
|
||||
self.market
|
||||
.compute(indexer, &self.prices, &self.indexes, &self.blocks, exit)
|
||||
.compute(indexer, &self.price, &self.indexes, &self.blocks, exit)
|
||||
})
|
||||
});
|
||||
|
||||
@@ -321,7 +321,7 @@ impl Computer {
|
||||
&self.indexes,
|
||||
&self.blocks,
|
||||
&self.inputs,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
exit,
|
||||
)
|
||||
})?;
|
||||
@@ -331,7 +331,7 @@ impl Computer {
|
||||
&self.indexes,
|
||||
&self.blocks,
|
||||
&self.transactions,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
exit,
|
||||
)
|
||||
})
|
||||
@@ -343,7 +343,7 @@ impl Computer {
|
||||
&self.indexes,
|
||||
&self.inputs,
|
||||
&self.blocks,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
exit,
|
||||
)
|
||||
})?;
|
||||
@@ -360,7 +360,7 @@ impl Computer {
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&self.blocks,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
&self.mining,
|
||||
exit,
|
||||
)
|
||||
@@ -372,7 +372,7 @@ impl Computer {
|
||||
self.investing.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
&self.blocks,
|
||||
&self.market.lookback,
|
||||
exit,
|
||||
@@ -388,7 +388,7 @@ impl Computer {
|
||||
&self.outputs,
|
||||
&self.transactions,
|
||||
&self.blocks,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
exit,
|
||||
)
|
||||
})?;
|
||||
@@ -421,7 +421,7 @@ impl Computer {
|
||||
&self.blocks,
|
||||
&self.mining,
|
||||
&self.transactions,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
&self.distribution,
|
||||
exit,
|
||||
)
|
||||
@@ -430,7 +430,7 @@ impl Computer {
|
||||
timed("Computed cointime", || {
|
||||
self.cointime.compute(
|
||||
indexer,
|
||||
&self.prices,
|
||||
&self.price,
|
||||
&self.blocks,
|
||||
&self.mining,
|
||||
&self.supply,
|
||||
@@ -445,7 +445,7 @@ impl Computer {
|
||||
|
||||
self.indicators
|
||||
.rarity_meter
|
||||
.compute(indexer, &self.distribution, &self.prices, exit)?;
|
||||
.compute(indexer, &self.distribution, &self.price, exit)?;
|
||||
|
||||
info!("Total compute time: {:?}", compute_start.elapsed());
|
||||
Ok(())
|
||||
@@ -498,7 +498,7 @@ impl_iter_named!(
|
||||
investing,
|
||||
market,
|
||||
pools,
|
||||
prices,
|
||||
price,
|
||||
distribution,
|
||||
supply,
|
||||
inputs,
|
||||
|
||||
@@ -4,13 +4,13 @@ use brk_types::{StoredF32, Timestamp};
|
||||
use vecdb::{Exit, ReadableVec, VecIndex};
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{indexes, prices};
|
||||
use crate::{indexes, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
indexes: &indexes::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -2,7 +2,7 @@ use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use crate::{blocks, indexes, prices};
|
||||
use crate::{blocks, indexes, price};
|
||||
|
||||
use super::Vecs;
|
||||
|
||||
@@ -10,7 +10,7 @@ impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
indexes: &indexes::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, prices};
|
||||
use crate::{blocks, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
|
||||
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, prices};
|
||||
use crate::{blocks, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
|
||||
@@ -4,13 +4,13 @@ use brk_types::{BasisPoints16, StoredF32};
|
||||
use vecdb::{Exit, ReadableVec, VecIndex};
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, prices};
|
||||
use crate::{blocks, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -5,14 +5,14 @@ use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{
|
||||
blocks, internal::RatioDiffDollarsBps32, investing::ByDcaPeriod, market::lookback, prices,
|
||||
blocks, internal::RatioDiffDollarsBps32, investing::ByDcaPeriod, market::lookback, price,
|
||||
};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
lookback: &lookback::Vecs,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -10,7 +10,7 @@ use super::{
|
||||
use crate::{
|
||||
blocks,
|
||||
internal::{RatioDollarsBp32, WindowsTo1m},
|
||||
prices,
|
||||
price,
|
||||
};
|
||||
|
||||
impl Vecs {
|
||||
@@ -19,7 +19,7 @@ impl Vecs {
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
returns: &returns::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
moving_average: &moving_average::Vecs,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::MacdChain;
|
||||
use crate::{blocks, prices};
|
||||
use crate::{blocks, price};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn compute(
|
||||
chain: &mut MacdChain,
|
||||
indexer: &Indexer,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
fast_days: usize,
|
||||
slow_days: usize,
|
||||
signal_days: usize,
|
||||
|
||||
@@ -3,7 +3,7 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, indexes, prices, transactions};
|
||||
use crate::{blocks, indexes, price, transactions};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -13,7 +13,7 @@ impl Vecs {
|
||||
indexes: &indexes::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
transactions: &transactions::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.db.sync_bg_tasks()?;
|
||||
|
||||
@@ -7,7 +7,7 @@ use super::Vecs;
|
||||
use crate::{
|
||||
blocks, indexes,
|
||||
internal::{RatioDollarsBp32, RatioSatsBp16},
|
||||
prices, transactions,
|
||||
price, transactions,
|
||||
};
|
||||
|
||||
impl Vecs {
|
||||
@@ -18,7 +18,7 @@ impl Vecs {
|
||||
indexes: &indexes::Vecs,
|
||||
lookback: &blocks::LookbackVecs,
|
||||
transactions: &transactions::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
|
||||
@@ -3,7 +3,7 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, indexes, inputs, prices};
|
||||
use crate::{blocks, indexes, inputs, price};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -13,7 +13,7 @@ impl Vecs {
|
||||
indexes: &indexes::Vecs,
|
||||
inputs: &inputs::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.db.sync_bg_tasks()?;
|
||||
|
||||
@@ -4,13 +4,13 @@ use brk_types::{Height, OutputType, Sats, TxOutIndex};
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
|
||||
|
||||
use super::Vecs;
|
||||
use crate::prices;
|
||||
use crate::price;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
|
||||
@@ -11,7 +11,7 @@ use crate::{
|
||||
MaskSats, PercentRollingWindows, RatioU64Bp16, ValuePerBlockCumulativeRolling,
|
||||
WindowStartVec, Windows,
|
||||
},
|
||||
mining, prices,
|
||||
mining, price,
|
||||
};
|
||||
|
||||
use super::minor;
|
||||
@@ -63,7 +63,7 @@ impl Vecs {
|
||||
indexer: &Indexer,
|
||||
pool: &impl ReadableVec<Height, PoolSlug>,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
mining: &mining::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::{
|
||||
WindowStartVec, Windows,
|
||||
db_utils::{finalize_db, open_db},
|
||||
},
|
||||
mining, prices,
|
||||
mining, price,
|
||||
};
|
||||
|
||||
pub const DB_NAME: &str = "pools";
|
||||
@@ -90,7 +90,7 @@ impl Vecs {
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
mining: &mining::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
+89
-47
@@ -2,7 +2,9 @@ use std::ops::Range;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_indexer::{Indexer, Lengths};
|
||||
use brk_oracle::{Config, Histogram, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin};
|
||||
use brk_oracle::{
|
||||
bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW,
|
||||
};
|
||||
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use tracing::info;
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, StorageMode, VecIndex, WritableVec};
|
||||
@@ -61,8 +63,8 @@ impl Vecs {
|
||||
fn compute_prices(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
|
||||
let source_version = indexer.vecs.outputs.value.version()
|
||||
+ indexer.vecs.outputs.output_type.version();
|
||||
let source_version =
|
||||
indexer.vecs.outputs.value.version() + indexer.vecs.outputs.output_type.version();
|
||||
self.spot
|
||||
.cents
|
||||
.height
|
||||
@@ -71,7 +73,7 @@ impl Vecs {
|
||||
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
if total_heights <= START_HEIGHT {
|
||||
if total_heights <= START_HEIGHT_SLOW {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -83,17 +85,12 @@ impl Vecs {
|
||||
.inner
|
||||
.truncate_if_needed_at(truncate_to)?;
|
||||
|
||||
if self.spot.cents.height.len() < START_HEIGHT {
|
||||
for line in brk_oracle::PRICES
|
||||
.lines()
|
||||
.skip(self.spot.cents.height.len())
|
||||
{
|
||||
if self.spot.cents.height.len() >= START_HEIGHT {
|
||||
if self.spot.cents.height.len() < START_HEIGHT_SLOW {
|
||||
for cents in brk_oracle::pre_oracle_prices_from(self.spot.cents.height.len()) {
|
||||
if self.spot.cents.height.len() >= START_HEIGHT_SLOW {
|
||||
break;
|
||||
}
|
||||
let dollars: f64 = line.parse().unwrap_or(0.0);
|
||||
let cents = (dollars * 100.0).round() as u64;
|
||||
self.spot.cents.height.inner.push(Cents::new(cents));
|
||||
self.spot.cents.height.inner.push(cents);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,8 +98,8 @@ impl Vecs {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let config = Config::default();
|
||||
let committed = self.spot.cents.height.len();
|
||||
let config = Config::for_height(committed);
|
||||
let prev_cents = self
|
||||
.spot
|
||||
.cents
|
||||
@@ -110,9 +107,9 @@ impl Vecs {
|
||||
.collect_one_at(committed - 1)
|
||||
.unwrap();
|
||||
let seed_bin = cents_to_bin(prev_cents.inner() as f64);
|
||||
let warmup = config.window_size.min(committed - START_HEIGHT);
|
||||
let warmup = config.window_size.min(committed - START_HEIGHT_SLOW);
|
||||
let mut oracle = Oracle::from_checkpoint(seed_bin, config, |o| {
|
||||
Self::feed_blocks(o, indexer, (committed - warmup)..committed, None);
|
||||
Self::feed_blocks_for_warmup(o, indexer, (committed - warmup)..committed, None);
|
||||
});
|
||||
|
||||
let num_new = total_heights - committed;
|
||||
@@ -121,19 +118,48 @@ impl Vecs {
|
||||
committed, total_heights
|
||||
);
|
||||
|
||||
let ref_bins =
|
||||
Self::feed_blocks(&mut oracle, indexer, committed..total_heights, None);
|
||||
// Slow cold-start EMA up to START_HEIGHT_FAST, then switch to the fast
|
||||
// mature-market EMA. Steady-state runs start past START_HEIGHT_FAST and skip
|
||||
// the slow segment entirely.
|
||||
{
|
||||
let mut processed = 0usize;
|
||||
let mut push_ref_bin = |ref_bin| {
|
||||
self.spot
|
||||
.cents
|
||||
.height
|
||||
.inner
|
||||
.push(Cents::new(bin_to_cents(ref_bin)));
|
||||
|
||||
for (i, ref_bin) in ref_bins.into_iter().enumerate() {
|
||||
self.spot
|
||||
.cents
|
||||
.height
|
||||
.inner
|
||||
.push(Cents::new(bin_to_cents(ref_bin)));
|
||||
processed += 1;
|
||||
let progress = (processed * 100 / num_new) as u8;
|
||||
if processed > 1 && progress > (((processed - 1) * 100 / num_new) as u8) {
|
||||
info!("Oracle price computation: {}%", progress);
|
||||
}
|
||||
};
|
||||
|
||||
let progress = ((i + 1) * 100 / num_new) as u8;
|
||||
if i > 0 && progress > ((i * 100 / num_new) as u8) {
|
||||
info!("Oracle price computation: {}%", progress);
|
||||
if committed < START_HEIGHT_FAST {
|
||||
let slow_end = START_HEIGHT_FAST.min(total_heights);
|
||||
Self::feed_blocks_with(
|
||||
&mut oracle,
|
||||
indexer,
|
||||
committed..slow_end,
|
||||
None,
|
||||
|_, _, ref_bin| push_ref_bin(ref_bin),
|
||||
);
|
||||
if slow_end == START_HEIGHT_FAST {
|
||||
oracle.reconfigure(Config::default());
|
||||
}
|
||||
}
|
||||
|
||||
let fast_start = committed.max(START_HEIGHT_FAST);
|
||||
if fast_start < total_heights {
|
||||
Self::feed_blocks_with(
|
||||
&mut oracle,
|
||||
indexer,
|
||||
fast_start..total_heights,
|
||||
None,
|
||||
|_, _, ref_bin| push_ref_bin(ref_bin),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,10 +179,6 @@ impl Vecs {
|
||||
/// Feed a range of blocks from the indexer into an Oracle (skipping coinbase),
|
||||
/// returning per-block ref_bin values.
|
||||
///
|
||||
/// A transaction carrying an `OP_RETURN` output is protocol machinery, not a
|
||||
/// dollar-denominated payment, so all of its outputs are dropped from the
|
||||
/// histogram. This needs per-transaction grouping of a block's outputs.
|
||||
///
|
||||
/// Pass `cap = None` from compute paths, when the indexer is quiescent and
|
||||
/// raw vec lengths are authoritative. Pass `cap = Some(&safe_lengths)` from
|
||||
/// reader paths so concurrent writer pushes past the cap are invisible.
|
||||
@@ -166,6 +188,33 @@ impl Vecs {
|
||||
range: Range<usize>,
|
||||
cap: Option<&Lengths>,
|
||||
) -> Vec<f64> {
|
||||
let mut ref_bins = Vec::with_capacity(range.len());
|
||||
Self::feed_blocks_with(oracle, indexer, range, cap, |_, _, ref_bin| {
|
||||
ref_bins.push(ref_bin);
|
||||
});
|
||||
ref_bins
|
||||
}
|
||||
|
||||
/// Feed blocks into an Oracle when callers only need the warmed EMA/window state.
|
||||
pub fn feed_blocks_for_warmup<IM: StorageMode>(
|
||||
oracle: &mut Oracle,
|
||||
indexer: &Indexer<IM>,
|
||||
range: Range<usize>,
|
||||
cap: Option<&Lengths>,
|
||||
) {
|
||||
Self::feed_blocks_with(oracle, indexer, range, cap, |_, _, _| {});
|
||||
}
|
||||
|
||||
/// Feed a range of blocks into an Oracle and call `on_block` after each
|
||||
/// processed block. This lets callers observe derived state such as EMA
|
||||
/// without duplicating the histogram extraction path.
|
||||
pub fn feed_blocks_with<IM: StorageMode>(
|
||||
oracle: &mut Oracle,
|
||||
indexer: &Indexer<IM>,
|
||||
range: Range<usize>,
|
||||
cap: Option<&Lengths>,
|
||||
mut on_block: impl FnMut(usize, &Oracle, f64),
|
||||
) {
|
||||
let (total_txs, total_outputs, height_len) = match cap {
|
||||
Some(c) => (
|
||||
c.tx_index.to_usize(),
|
||||
@@ -193,8 +242,6 @@ impl Vecs {
|
||||
.first_txout_index
|
||||
.collect_range_at(range.start, collect_end);
|
||||
|
||||
let mut ref_bins = Vec::with_capacity(range.len());
|
||||
|
||||
// Cursor avoids per-block PcoVec page decompression for the
|
||||
// tx-indexed first_txout_index lookup. Accessed tx_index values
|
||||
// are strictly increasing across blocks, so it only advances forward.
|
||||
@@ -239,26 +286,21 @@ impl Vecs {
|
||||
&mut output_types,
|
||||
);
|
||||
|
||||
let mut hist = Histogram::zeros();
|
||||
for tx in 0..tx_count {
|
||||
let tx_outputs = (0..tx_count).map(|tx| {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
if output_types[lo..hi].contains(&OutputType::OpReturn) {
|
||||
continue;
|
||||
}
|
||||
for i in lo..hi {
|
||||
if let Some(bin) = oracle.output_to_bin(values[i], output_types[i]) {
|
||||
hist.increment(bin);
|
||||
}
|
||||
}
|
||||
}
|
||||
values[lo..hi]
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(output_types[lo..hi].iter().copied())
|
||||
});
|
||||
let hist = PaymentFilter::for_height(range.start + idx).histogram(tx_outputs);
|
||||
|
||||
ref_bins.push(oracle.process_histogram(&hist));
|
||||
let ref_bin = oracle.process_histogram(&hist);
|
||||
on_block(range.start + idx, oracle, ref_bin);
|
||||
}
|
||||
|
||||
ref_bins
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,7 @@ use crate::{
|
||||
use by_unit::{OhlcByUnit, PriceByUnit, SplitByUnit, SplitCloseByUnit, SplitIndexesByUnit};
|
||||
use ohlcs::{LazyOhlcVecs, OhlcVecs};
|
||||
|
||||
pub const DB_NAME: &str = "prices";
|
||||
pub const DB_NAME: &str = "price";
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Vecs<M: StorageMode = Rw> {
|
||||
@@ -4,7 +4,7 @@ use brk_types::Sats;
|
||||
use vecdb::{Exit, VecIndex};
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{mining, outputs, prices};
|
||||
use crate::{mining, outputs, price};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
@@ -12,7 +12,7 @@ impl Vecs {
|
||||
indexer: &Indexer,
|
||||
outputs: &outputs::Vecs,
|
||||
mining: &mining::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
|
||||
@@ -7,7 +7,7 @@ use vecdb::Exit;
|
||||
const INITIAL_SUBSIDY: f64 = Sats::ONE_BTC_U64 as f64 * 50.0;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, distribution, mining, outputs, prices, transactions};
|
||||
use crate::{blocks, distribution, mining, outputs, price, transactions};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -18,7 +18,7 @@ impl Vecs {
|
||||
blocks: &blocks::Vecs,
|
||||
mining: &mining::Vecs,
|
||||
transactions: &transactions::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
distribution: &distribution::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
|
||||
@@ -3,7 +3,7 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, indexes, inputs, prices};
|
||||
use crate::{blocks, indexes, inputs, price};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -13,7 +13,7 @@ impl Vecs {
|
||||
indexes: &indexes::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
inputs: &inputs::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.db.sync_bg_tasks()?;
|
||||
|
||||
@@ -5,7 +5,7 @@ use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::transactions::{count, fees};
|
||||
use crate::{indexes, internal::Windows, prices};
|
||||
use crate::{indexes, internal::Windows, price};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -13,7 +13,7 @@ impl Vecs {
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
prices: &prices::Vecs,
|
||||
prices: &price::Vecs,
|
||||
count_vecs: &count::Vecs,
|
||||
fees_vecs: &fees::Vecs,
|
||||
exit: &Exit,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Mempool info + price-blending output histogram.
|
||||
|
||||
use brk_oracle::Histogram;
|
||||
use brk_oracle::HistogramRaw;
|
||||
use brk_types::MempoolInfo;
|
||||
|
||||
use crate::Mempool;
|
||||
@@ -11,13 +11,21 @@ impl Mempool {
|
||||
self.read().info.clone()
|
||||
}
|
||||
|
||||
/// Snapshot of pre-bucketed oracle bins across all live mempool tx
|
||||
/// outputs. The total is maintained incrementally by `TxStore` on
|
||||
/// every insert/remove, so this hot path is `O(NUM_BINS)` regardless
|
||||
/// of pool size. Used by `live_price` to blend the mempool into the
|
||||
/// committed oracle without re-parsing scripts per request.
|
||||
/// Snapshot of pre-bucketed round-dollar-eligible bins across all live
|
||||
/// mempool tx outputs. Maintained incrementally by `TxStore` on every
|
||||
/// insert/remove, so this hot path is `O(NUM_BINS)` regardless of pool
|
||||
/// size. Used by `live_price` to blend the mempool into the committed
|
||||
/// oracle without re-parsing scripts per request.
|
||||
#[must_use]
|
||||
pub fn live_histogram(&self) -> Histogram {
|
||||
self.read().txs.live_histogram()
|
||||
pub fn live_eligible_histogram(&self) -> HistogramRaw {
|
||||
self.read().txs.live_eligible_histogram()
|
||||
}
|
||||
|
||||
/// Snapshot of the raw histogram: every live mempool output binned by
|
||||
/// value with no payment filtering. Backs the `histogram/raw/live`
|
||||
/// endpoint.
|
||||
#[must_use]
|
||||
pub fn live_raw_histogram(&self) -> HistogramRaw {
|
||||
self.read().txs.live_raw_histogram()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
use brk_oracle::{sats_to_bin, HistogramRaw, PaymentFilter};
|
||||
use brk_types::Transaction;
|
||||
|
||||
use crate::stores::tx_store::TxRecord;
|
||||
|
||||
/// The two live per-bin histograms the pool maintains incrementally as txs
|
||||
/// enter and leave: `eligible` applies the round-dollar payment filter (it
|
||||
/// feeds the oracle blend), `raw` bins every output by value with no filtering.
|
||||
/// Add and remove run through the same code so the two stay symmetric.
|
||||
#[derive(Default)]
|
||||
pub struct LiveHistograms {
|
||||
eligible: HistogramRaw,
|
||||
raw: HistogramRaw,
|
||||
}
|
||||
|
||||
impl LiveHistograms {
|
||||
/// Fold a record's outputs into both histograms.
|
||||
pub fn add(&mut self, record: &TxRecord) {
|
||||
Self::eligible_bins(&record.tx, |bin| self.eligible[bin as usize] += 1);
|
||||
for bin in Self::raw_bins(&record.tx) {
|
||||
self.raw[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Reverse a previous `add` for the same record.
|
||||
pub fn remove(&mut self, record: &TxRecord) {
|
||||
Self::eligible_bins(&record.tx, |bin| self.eligible[bin as usize] -= 1);
|
||||
for bin in Self::raw_bins(&record.tx) {
|
||||
self.raw[bin] -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Round-dollar-eligible bins, blended into the oracle by `live_price`.
|
||||
pub fn eligible(&self) -> HistogramRaw {
|
||||
self.eligible.clone()
|
||||
}
|
||||
|
||||
/// Every live output binned by value, no payment filtering.
|
||||
pub fn raw(&self) -> HistogramRaw {
|
||||
self.raw.clone()
|
||||
}
|
||||
|
||||
/// Round-dollar-eligible bins, applying the oracle payment filter. Calls
|
||||
/// `emit(bin)` per eligible output. Deterministic over a tx's outputs,
|
||||
/// which are never mutated after insert, so add and remove recompute it
|
||||
/// identically rather than caching.
|
||||
fn eligible_bins(tx: &Transaction, emit: impl FnMut(u16)) {
|
||||
PaymentFilter::MODERN.for_each_bin(tx.output.iter().map(|o| (o.value, o.type_())), emit);
|
||||
}
|
||||
|
||||
/// Raw bin index per output, dropping only values outside the bin domain
|
||||
/// (zero / out-of-range).
|
||||
fn raw_bins(tx: &Transaction) -> impl Iterator<Item = usize> + '_ {
|
||||
tx.output.iter().filter_map(|o| sats_to_bin(o.value))
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,13 @@
|
||||
//! one lock-order discipline.
|
||||
|
||||
mod addr_tracker;
|
||||
mod live_histograms;
|
||||
mod outpoint_spends;
|
||||
mod output_bins;
|
||||
mod tx_graveyard;
|
||||
mod tx_store;
|
||||
|
||||
pub use addr_tracker::AddrTracker;
|
||||
pub use live_histograms::LiveHistograms;
|
||||
pub use outpoint_spends::OutpointSpends;
|
||||
pub use output_bins::OutputBins;
|
||||
pub use tx_graveyard::{TxGraveyard, TxTombstone};
|
||||
pub use tx_store::TxStore;
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
use brk_oracle::default_eligible_bin;
|
||||
use brk_types::Transaction;
|
||||
use smallvec::SmallVec;
|
||||
|
||||
/// Pre-bucketed oracle bins for a tx's eligible outputs. Computed once on
|
||||
/// insert so `Mempool::live_histogram` can bin all live outputs without
|
||||
/// re-parsing scripts or recomputing eligibility per request.
|
||||
pub struct OutputBins(SmallVec<[u16; 4]>);
|
||||
|
||||
impl OutputBins {
|
||||
pub fn from_tx(tx: &Transaction) -> Self {
|
||||
Self(
|
||||
tx.output
|
||||
.iter()
|
||||
.filter_map(|o| default_eligible_bin(o.value, o.type_()))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = u16> + '_ {
|
||||
self.0.iter().copied()
|
||||
}
|
||||
}
|
||||
@@ -1,28 +1,21 @@
|
||||
use brk_oracle::Histogram;
|
||||
use brk_oracle::HistogramRaw;
|
||||
use brk_types::{MempoolRecentTx, Transaction, TxOut, Txid, TxidPrefix, Vin};
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
|
||||
use crate::{state::TxEntry, stores::OutputBins};
|
||||
use crate::{state::TxEntry, stores::LiveHistograms};
|
||||
|
||||
const RECENT_CAP: usize = 10;
|
||||
|
||||
/// Per-tx record: live tx body, its mempool entry, and the pre-bucketed
|
||||
/// oracle bins for its outputs. Kept under one key so a single map probe
|
||||
/// returns everything readers need.
|
||||
/// Per-tx record: live tx body and its mempool entry, kept under one key
|
||||
/// so a single map probe returns everything readers need.
|
||||
pub struct TxRecord {
|
||||
pub tx: Transaction,
|
||||
pub entry: TxEntry,
|
||||
pub output_bins: OutputBins,
|
||||
}
|
||||
|
||||
impl TxRecord {
|
||||
pub fn new(tx: Transaction, entry: TxEntry) -> Self {
|
||||
let output_bins = OutputBins::from_tx(&tx);
|
||||
Self {
|
||||
tx,
|
||||
entry,
|
||||
output_bins,
|
||||
}
|
||||
Self { tx, entry }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,15 +25,15 @@ impl TxRecord {
|
||||
/// set of prefixes whose tx still has at least one `prevout: None`,
|
||||
/// maintained on every `insert` / `remove_by_prefix` / `apply_fills`
|
||||
/// so the post-update prevout filler can early-exit when empty.
|
||||
/// `live_histogram` mirrors the union of each record's `OutputBins`,
|
||||
/// kept in sync on `insert` / `remove_by_prefix` so the oracle-blend
|
||||
/// read path is a single array clone, not a full pool walk.
|
||||
/// `histograms` holds the eligible (oracle-blend) and raw per-bin output
|
||||
/// histograms, kept in sync on `insert` / `remove_by_prefix` so each read
|
||||
/// path is a single array clone, not a full pool walk.
|
||||
#[derive(Default)]
|
||||
pub struct TxStore {
|
||||
records: FxHashMap<TxidPrefix, TxRecord>,
|
||||
recent: Vec<MempoolRecentTx>,
|
||||
unresolved: FxHashSet<TxidPrefix>,
|
||||
live_histogram: Histogram,
|
||||
histograms: LiveHistograms,
|
||||
}
|
||||
|
||||
impl TxStore {
|
||||
@@ -92,9 +85,7 @@ impl TxStore {
|
||||
self.unresolved.insert(prefix);
|
||||
}
|
||||
let record = TxRecord::new(tx, entry);
|
||||
for bin in record.output_bins.iter() {
|
||||
self.live_histogram[bin as usize] += 1;
|
||||
}
|
||||
self.histograms.add(&record);
|
||||
self.records.insert(prefix, record);
|
||||
}
|
||||
|
||||
@@ -112,16 +103,21 @@ impl TxStore {
|
||||
pub fn remove_by_prefix(&mut self, prefix: &TxidPrefix) -> Option<TxRecord> {
|
||||
let record = self.records.remove(prefix)?;
|
||||
self.unresolved.remove(prefix);
|
||||
for bin in record.output_bins.iter() {
|
||||
self.live_histogram[bin as usize] -= 1;
|
||||
}
|
||||
self.histograms.remove(&record);
|
||||
Some(record)
|
||||
}
|
||||
|
||||
/// Snapshot the live oracle-bin histogram. Maintained incrementally
|
||||
/// on insert/remove, so this is `O(NUM_BINS)`, not `O(live_outputs)`.
|
||||
pub fn live_histogram(&self) -> Histogram {
|
||||
self.live_histogram.clone()
|
||||
/// Snapshot the round-dollar-eligible histogram that feeds the oracle
|
||||
/// blend. Maintained incrementally, so this is `O(NUM_BINS)`, not
|
||||
/// `O(live_outputs)`.
|
||||
pub fn live_eligible_histogram(&self) -> HistogramRaw {
|
||||
self.histograms.eligible()
|
||||
}
|
||||
|
||||
/// Snapshot the raw histogram: every live output binned by value with no
|
||||
/// payment filtering. Maintained incrementally alongside the eligible one.
|
||||
pub fn live_raw_histogram(&self) -> HistogramRaw {
|
||||
self.histograms.raw()
|
||||
}
|
||||
|
||||
/// Set of prefixes with at least one unfilled prevout. Used by the
|
||||
@@ -263,7 +259,10 @@ mod tests {
|
||||
assert_eq!(applied[0].value, new_prevout.value);
|
||||
|
||||
let record = store.record_by_prefix(&prefix).expect("record present");
|
||||
assert_eq!(record.tx.input[0].prevout.as_ref().unwrap().value, new_prevout.value);
|
||||
assert_eq!(
|
||||
record.tx.input[0].prevout.as_ref().unwrap().value,
|
||||
new_prevout.value
|
||||
);
|
||||
assert_eq!(
|
||||
record.tx.input[1].prevout.as_ref().unwrap().value,
|
||||
prev_present.value
|
||||
@@ -277,7 +276,10 @@ mod tests {
|
||||
let stray_prefix = TxidPrefix::from(&fake_txid(0xFF));
|
||||
let applied = store.apply_fills(
|
||||
&stray_prefix,
|
||||
vec![(Vin::from(0u32), TxOut::from((ScriptBuf::new(), Sats::from(1u64))))],
|
||||
vec![(
|
||||
Vin::from(0u32),
|
||||
TxOut::from((ScriptBuf::new(), Sats::from(1u64))),
|
||||
)],
|
||||
);
|
||||
assert!(applied.is_empty());
|
||||
}
|
||||
@@ -319,10 +321,7 @@ mod tests {
|
||||
let tx_a = fake_tx(
|
||||
20,
|
||||
&[Some(TxOut::from((p2wpkh_script(8), Sats::from(1_234u64))))],
|
||||
&[
|
||||
(p2wpkh_script(9), 2_345),
|
||||
(p2wpkh_script(10), 3_456),
|
||||
],
|
||||
&[(p2wpkh_script(9), 2_345), (p2wpkh_script(10), 3_456)],
|
||||
);
|
||||
let tx_b = fake_tx(
|
||||
21,
|
||||
@@ -335,11 +334,41 @@ mod tests {
|
||||
store.insert(tx_a, entry_a);
|
||||
store.insert(tx_b, entry_b);
|
||||
|
||||
let total_after_both: u32 = store.live_histogram().iter().sum();
|
||||
let total_after_both: u32 = store.live_eligible_histogram().iter().sum();
|
||||
assert_eq!(total_after_both, 3, "two outputs + one output");
|
||||
|
||||
store.remove_by_prefix(&prefix_a);
|
||||
let total_after_remove: u32 = store.live_histogram().iter().sum();
|
||||
let total_after_remove: u32 = store.live_eligible_histogram().iter().sum();
|
||||
assert_eq!(total_after_remove, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_histogram_bins_outputs_the_eligible_filter_drops() {
|
||||
let mut store = TxStore::default();
|
||||
// 2_345 sats is a round-dollar-eligible payment; 100_000_000 sats (1 BTC)
|
||||
// is a round-BTC value the eligible filter drops but raw still bins.
|
||||
let tx = fake_tx(
|
||||
30,
|
||||
&[Some(TxOut::from((p2wpkh_script(1), Sats::from(50_000u64))))],
|
||||
&[(p2wpkh_script(2), 2_345), (p2wpkh_script(3), 100_000_000)],
|
||||
);
|
||||
let entry = entry_for(&tx, 100, 100);
|
||||
let prefix = entry.txid_prefix();
|
||||
store.insert(tx, entry);
|
||||
|
||||
assert_eq!(
|
||||
store.live_eligible_histogram().iter().sum::<u32>(),
|
||||
1,
|
||||
"round-BTC output filtered out of the eligible histogram"
|
||||
);
|
||||
assert_eq!(
|
||||
store.live_raw_histogram().iter().sum::<u32>(),
|
||||
2,
|
||||
"raw histogram bins every output"
|
||||
);
|
||||
|
||||
store.remove_by_prefix(&prefix);
|
||||
assert_eq!(store.live_eligible_histogram().iter().sum::<u32>(), 0);
|
||||
assert_eq!(store.live_raw_histogram().iter().sum::<u32>(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
+61
-37
@@ -1,8 +1,8 @@
|
||||
# brk_oracle
|
||||
|
||||
**Version 2**
|
||||
**Version 3**
|
||||
|
||||
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 525,000 (May 2018) onward.
|
||||
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 340,000 (January 2015) onward.
|
||||
|
||||
Inspired by [UTXOracle](https://utxo.live/oracle/) by [@SteveSimple](https://x.com/SteveSimple), which proved the concept. brk_oracle takes the same core insight and redesigns the algorithm for per-block resolution and rolling operation. See [comparison](#comparison-with-utxoracle) below.
|
||||
|
||||
@@ -42,13 +42,13 @@ The spacing between spikes is constant (set by the ratios between dollar amounts
|
||||
|
||||
## How it works
|
||||
|
||||
The oracle tracks the price incrementally, block by block, starting from a known seed price. Each new block nudges the estimate. The search window is narrow (about ±10 bins, or ±12%), so the oracle can only follow gradual movement — it cannot jump to an arbitrary price from scratch. This is by design: it makes the algorithm resistant to noise.
|
||||
The oracle tracks the price incrementally, block by block, starting from a known seed price. Each new block nudges the estimate. The search window is narrow (about 12 bins, or +15% / -12% in price), so the oracle can only follow gradual movement, not jump to an arbitrary price from scratch. This is by design: it makes the algorithm resistant to noise.
|
||||
|
||||
For each new block:
|
||||
|
||||
### 1. Filter outputs
|
||||
|
||||
Skip the coinbase transaction, and skip every output of a transaction carrying an `OP_RETURN`: that transaction is protocol machinery, not a dollar-denominated payment, so its payout amounts are not price signal. Then exclude noisy outputs: script types dominated by protocol activity (P2TR by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
|
||||
Skip the coinbase transaction, and skip every output of a transaction carrying an `OP_RETURN`: that transaction is protocol machinery, not a dollar-denominated payment, so its payout amounts are not price signal. Below height 630,000, also skip every output of a transaction with more than 100 outputs: a large fan-out is a batch payout (exchange sweep, mixer), not a round-dollar payment, and the thin early signal needs it removed. At and above height 630,000, the transaction fan-out cap relaxes to 250 outputs so dense-chain payment activity remains visible while very large fan-outs cannot dominate one EMA slot. Then exclude noisy outputs: script types dominated by protocol activity (P2TR by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
|
||||
|
||||
### 2. Build a log-scale histogram
|
||||
|
||||
@@ -87,7 +87,7 @@ The fixed ratios between round-dollar amounts ($1, $2, $3, $5, ... $10,000) crea
|
||||
The oracle slides this stencil across the EMA histogram within the search window. At each candidate position:
|
||||
|
||||
1. **Read** the EMA value at all 19 expected spike locations
|
||||
2. **Normalize** each value by dividing by that offset's peak within the search window — this gives rare amounts like $3 equal voting weight to common amounts like $100
|
||||
2. **Normalize** each value by dividing by that offset's peak within the search window: this gives rare amounts like $3 equal voting weight to common amounts like $100
|
||||
3. **Sum** the 19 normalized values into a single score
|
||||
|
||||
The position with the highest score is where the fingerprint best matches the histogram.
|
||||
@@ -102,7 +102,7 @@ A $100 purchase at price P produces `$100 / P × 10⁸` sats, which lands in bin
|
||||
= (10 − log₁₀(P)) × 200
|
||||
```
|
||||
|
||||
So the stencil's winning position — the bin where $100 purchases land — directly encodes the price:
|
||||
So the stencil's winning position, the bin where $100 purchases land, directly encodes the price:
|
||||
|
||||
```
|
||||
price = 10^(10 − bin / 200) dollars
|
||||
@@ -122,9 +122,9 @@ Parabolic interpolation between the best bin and its two neighbors refines the e
|
||||
|
||||
The oracle consumes one pre-built histogram per block via `process_histogram(&hist)`, a `[u32; 2400]` bin-count array, and returns the updated reference bin.
|
||||
|
||||
The caller does the filtering when it builds the histogram. For each block it skips the coinbase, drops every output of a transaction carrying an `OP_RETURN`, then bins the rest. `default_eligible_bin(sats, output_type)` (or `Oracle::output_to_bin` for a non-default `Config`) applies the per-output rules: excluded script types, dust, and round-BTC values. It returns the bin index, or `None` for a filtered output.
|
||||
The caller filters as it builds the histogram, applying the [step 1](#1-filter-outputs) rules. `PaymentFilter::for_height(height).histogram(txs)` builds a fresh block histogram from non-coinbase transaction outputs. Incremental live callers use `PaymentFilter::MODERN.for_each_bin(outputs, emit)`, which applies the modern fan-out cap without requiring a height. `PaymentFilter::eligible_bin(sats, output_type)` returns an individual output's bin index, or `None` if filtered. The transaction-level rules include the OP_RETURN drop, the >100 transaction-output fan-out cap below height 630,000, and the >250 cap from height 630,000 onward.
|
||||
|
||||
The initial seed must be close to the real price at the starting height. The crate includes a `PRICES` constant with exchange prices for every height up to 630,000 to derive a seed from.
|
||||
The initial seed must be close to the real price at the starting height. The crate includes typed pre-oracle helpers for exchange prices at heights 0..340,000. `Oracle::from_seed()` uses the last baked price, height 339,999 (one below `START_HEIGHT_SLOW`), and the slow cold-start config to seed the oracle's first on-chain computation at height 340,000.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -134,10 +134,12 @@ All parameters via `Config` with sensible defaults:
|
||||
|-----------|---------|---------|
|
||||
| `alpha` | 2/7 | EMA decay rate (~6-block span) |
|
||||
| `window_size` | 12 | Ring buffer depth in blocks |
|
||||
| `search_below` / `search_above` | 9 / 11 | Search window around previous estimate (bins) |
|
||||
| `min_sats` | 1,000 | Dust threshold |
|
||||
| `exclude_common_round_values` | true | Filter d × 10ⁿ (d ∈ {1,2,3,5,6}) to prevent false stencil matches |
|
||||
| `excluded_output_types` | P2TR | Script types dominated by protocol activity |
|
||||
| `search_below` / `search_above` | 12 / 11 | Search window around previous estimate (bins) |
|
||||
| `shape_weight` | 0 | Shape-anchoring restoring-force weight. 0 disables it. `Config::slow()` sets 8 for the cold-start |
|
||||
|
||||
The output-filtering rules (1,000-sat dust floor, excluded P2TR, round-BTC exclusion) are not `Config` parameters: they are constants in the `filter` module so the indexer, per-request reconstruction, and mempool all bin identically. See [Input](#input).
|
||||
|
||||
Between heights 340,000 and 508,000 the oracle runs a slower cold-start configuration (`Config::slow()`: `alpha` = 0.10, ~19-block span, `window_size` = 40, `shape_weight` = 8). In the thin pre-2018 output mix the fast default octave-locks onto the round-dollar half-price pattern, so the slow EMA and the shape-anchoring restoring force resist that drift. At 508,000 `Oracle::reconfigure` switches to the defaults above (`shape_weight` back to 0), and `Config::for_height` returns the right one for any height.
|
||||
|
||||
## Comparison with UTXOracle
|
||||
|
||||
@@ -145,35 +147,36 @@ All parameters via `Config` with sensible defaults:
|
||||
|
||||
| | brk_oracle | UTXOracle |
|
||||
|---|---|---|
|
||||
| Resolution | Per-block (~10 min) + daily candles | Per-run consensus price + per-output intraday scatter |
|
||||
| Resolution | Per-block (~10 min); daily OHLC built downstream | Per-run consensus price + per-output intraday scatter |
|
||||
| Operation | Rolling: EMA over ring buffer, updates each block | Batch: processes a full day from scratch, stateless |
|
||||
| Algorithm | Single-pass stencil scoring with per-offset normalization | Multi-step: dual stencil → rough estimate → output-to-USD mapping → iterative convergence |
|
||||
| Steps to compute price | 7 (filter+bin → ring insert → EMA → per-offset peaks → score → argmax+parabolic → bin→price) | 10 (filter+bin → clip → smooth round BTC → sum → normalize → cap extremes → dual-stencil slide → neighbor weight-avg → output-to-USD map → iterative central price) |
|
||||
| Stencil | 19 round-USD offsets ($1 to $10k), each normalized to its own peak | 803-point Gaussian + weighted spike template targeting 17 round-USD amounts |
|
||||
| Round BTC handling | Excluded from histogram entirely | Histogram bins smoothed by averaging neighbors |
|
||||
| Output filtering | Per-tx OP_RETURN drop, then per-output: script type, dust threshold, round BTC | Per-tx: exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
|
||||
| Validated from | Height 525,000 (May 2018) | December 2023 |
|
||||
| Output filtering | Per-tx OP_RETURN drop, then per-output: script type, dust threshold, round BTC | Per-tx: not coinbase, no OP_RETURN, exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
|
||||
| Validated from | Height 340,000 (January 2015) | Dec 15, 2023 |
|
||||
| Language | Rust | Python |
|
||||
| Dependencies | None (pure computation, caller provides block data) | Bitcoin Core RPC |
|
||||
| Dependencies | None (pure computation, caller provides block data) | bitcoin-cli + direct blk file reads |
|
||||
| Bins per decade | 200 | 200 |
|
||||
|
||||
## Accuracy
|
||||
|
||||
Tested over 411,251 blocks (heights 525,000 to 949,800, as of May 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
|
||||
Tested over 596,251 exchange-covered blocks after running the oracle from height 340,000 through height 952,314. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
|
||||
|
||||
### Per-block
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Median error | 0.11% |
|
||||
| 95th percentile | 0.67% |
|
||||
| 99th percentile | 1.7% |
|
||||
| 99.9th percentile | 5.4% |
|
||||
| RMSE | 0.50% |
|
||||
| Max error | 33.4% |
|
||||
| Bias | +0.00 bins (essentially zero) |
|
||||
| Blocks > 5% error | 472 (0.11%) |
|
||||
| Blocks > 10% error | 177 |
|
||||
| Blocks > 20% error | 3 |
|
||||
| Median error | 0.15% |
|
||||
| 95th percentile | 1.2% |
|
||||
| 99th percentile | 3.4% |
|
||||
| 99.9th percentile | 15.6% |
|
||||
| RMSE | 0.97% |
|
||||
| Max error | 33.8% |
|
||||
| Bias | +0.06 bins (essentially zero) |
|
||||
| Blocks > 5% error | 3,233 (0.542%) |
|
||||
| Blocks > 10% error | 1,323 |
|
||||
| Blocks > 20% error | 154 |
|
||||
|
||||
### Daily candles
|
||||
|
||||
@@ -181,36 +184,57 @@ Oracle daily OHLC built from per-block prices vs exchange daily OHLC:
|
||||
|
||||
| | Median | RMSE | Max |
|
||||
|-------|--------|------|-----|
|
||||
| Open | 0.21% | 0.65% | 15.3% |
|
||||
| High | 0.53% | 1.12% | 28.0% |
|
||||
| Low | 0.51% | 1.38% | 19.7% |
|
||||
| Close | 0.24% | 0.73% | 15.4% |
|
||||
| Open | 0.24% | 1.07% | 29.1% |
|
||||
| High | 0.58% | 1.47% | 27.3% |
|
||||
| Low | 0.53% | 1.95% | 55.1% |
|
||||
| Close | 0.27% | 1.18% | 29.2% |
|
||||
|
||||
### By year
|
||||
|
||||
| Year | Blocks | Median | RMSE | Max | >5% | >10% | >20% | Price range |
|
||||
|------|--------|--------|------|-----|-----|------|------|-------------|
|
||||
| 2018 | 31,492 | 0.21% | 1.11% | 33.4% | 169 | 109 | 3 | $3,129–$8,488 |
|
||||
| 2019 | 54,272 | 0.16% | 0.69% | 17.4% | 165 | 53 | 0 | $3,338–$13,868 |
|
||||
| 2020 | 53,102 | 0.10% | 0.44% | 12.6% | 70 | 6 | 0 | $3,858–$29,322 |
|
||||
| 2015 | 51,249 | 0.26% | 1.67% | 33.8% | 916 | 449 | 25 | $198–$500 |
|
||||
| 2016 | 54,753 | 0.33% | 0.80% | 16.9% | 150 | 33 | 0 | $351–$989 |
|
||||
| 2017 | 55,959 | 0.45% | 2.05% | 28.6% | 1,527 | 606 | 67 | $0–$19,892 |
|
||||
| 2018 | 54,531 | 0.18% | 1.31% | 31.6% | 411 | 207 | 62 | $3,129–$17,178 |
|
||||
| 2019 | 54,272 | 0.16% | 0.59% | 17.4% | 100 | 16 | 0 | $3,338–$13,868 |
|
||||
| 2020 | 53,102 | 0.10% | 0.42% | 11.6% | 61 | 3 | 0 | $3,858–$29,322 |
|
||||
| 2021 | 52,733 | 0.07% | 0.47% | 14.4% | 42 | 9 | 0 | $27,678–$69,000 |
|
||||
| 2022 | 53,230 | 0.07% | 0.32% | 6.8% | 10 | 0 | 0 | $15,460–$48,240 |
|
||||
| 2023 | 54,032 | 0.10% | 0.25% | 6.6% | 5 | 0 | 0 | $16,490–$44,700 |
|
||||
| 2024 | 53,367 | 0.10% | 0.28% | 6.7% | 7 | 0 | 0 | $38,555–$108,298 |
|
||||
| 2025 | 53,113 | 0.11% | 0.25% | 5.8% | 4 | 0 | 0 | $74,409–$126,198 |
|
||||
| 2026 | 5,910 | 0.11% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000–$97,900 |
|
||||
| 2026 | 5,910 | 0.10% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000–$97,900 |
|
||||
|
||||
The oracle is only as good as the signal it reads. The largest errors cluster in late 2018: the November price crash fell faster than the narrow search window could follow (33% max error), and on-chain volume was lower then, so the round-dollar pattern was weaker (1.1% RMSE for the year). By 2020 the signal is strong enough for 0.1% median accuracy, and since 2022 no block exceeds 10% error.
|
||||
The oracle is only as good as the signal it reads. The largest errors cluster in the early cold-start, where thin 2015 on-chain volume gives a weaker round-dollar pattern: the 33.8% max error sits at height 341,498 (oracle ~$287 vs exchange ~$213) during the first weeks of warm-up. A second cluster sits just below the 508,000 regime switch, where the slow EMA lagged the fast early-2018 rally (~31.6% at height 507,278, oracle ~$6,685 vs exchange ~$8,800) before handing off to the fast default. The thin pre-2018 mix means 2015, 2017, and 2018 carry the bulk of the error (1.67%, 2.05%, and 1.31% RMSE). From 2019 the signal strengthens: by 2020 the oracle reaches 0.1% median accuracy, and since 2022 no block exceeds 10% error.
|
||||
|
||||
### Why no outlier smoothing?
|
||||
|
||||
Post-hoc smoothing — for example, correcting any block whose price deviates more than 5% from both its neighbors — would improve the aggregate numbers. This is deliberately not done, for two reasons:
|
||||
Post-hoc smoothing, for example correcting any block whose price deviates more than 5% from both its neighbors, would improve the aggregate numbers. This is deliberately not done, for two reasons:
|
||||
|
||||
1. **Simplicity**: The oracle is a single forward pass with no lookback corrections. Adding smoothing means defining thresholds, neighbor windows, and replacement strategies, all of which add complexity for marginal gain.
|
||||
2. **Finality**: Each block's price is produced once and never revised (unless the block itself is reorged). Downstream consumers can treat the oracle output as append-only. Smoothing would require retroactively changing already-published prices, breaking that property.
|
||||
|
||||
## Changelog
|
||||
|
||||
### v4
|
||||
|
||||
Changes from v3:
|
||||
|
||||
- **Modern fan-out cap**: below height 630,000 the oracle keeps the strict >100-output transaction drop introduced in v3. At and above 630,000 the cap now relaxes to 250 outputs instead of being fully lifted. This preserves dense-chain payment signal while preventing very large modern fan-outs from dominating a single EMA slot and creating a transient false round-dollar ladder.
|
||||
|
||||
`VERSION` is bumped to 4 so downstream consumers invalidate prices computed by an earlier algorithm.
|
||||
|
||||
### v3
|
||||
|
||||
Changes from v2:
|
||||
|
||||
- **Earlier start with a cold-start regime**: on-chain tracking begins at height 340,000 (January 2015) instead of 525,000, adding about 185,000 blocks of history. Below height 508,000 the oracle runs a slower EMA (`Config::slow()`, ~19-block span, window 40) paired with a shape-anchoring restoring force (`shape_weight` 8) that pulls candidate scores toward a slowly-adapted profile of the round-dollar arm shape, resisting the half-price octave drift the fast default locks onto in the thinner pre-2018 output mix. At height 508,000 it switches to the fast default via `Oracle::reconfigure`, which restores `shape_weight` to 0 and turns the force off.
|
||||
- **Max-outputs filter**: a transaction with more than 100 outputs is dropped from the histogram below height 630,000. Large fan-outs (exchange sweeps, mixer payouts) are batch machinery, not round-dollar payments, and the thin 2018-2020 signal needs them removed to stay locked onto the pattern. Above 630,000 on-chain volume is dense enough that the cap removes more genuine signal than noise, so it is lifted.
|
||||
- **Wider up-reach**: `search_below` raised from 9 to 12 bins. The sharp 2018 reversal candles need extra room to follow a fast move upward in price.
|
||||
|
||||
`VERSION` is bumped to 3 so downstream consumers invalidate prices computed by an earlier algorithm.
|
||||
|
||||
### v2
|
||||
|
||||
Changes from v1:
|
||||
|
||||
@@ -1,295 +0,0 @@
|
||||
//! Compare specific digit filter configurations across multiple start heights.
|
||||
//!
|
||||
//! Run with: cargo run -p brk_oracle --example compare_digits --release
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{Config, Histogram, NUM_BINS, Oracle, PRICES, cents_to_bin, sats_to_bin};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
const BINS_5PCT: f64 = 4.24;
|
||||
const BINS_10PCT: f64 = 8.28;
|
||||
const BINS_20PCT: f64 = 15.84;
|
||||
|
||||
fn bins_to_pct(bins: f64) -> f64 {
|
||||
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
|
||||
}
|
||||
|
||||
fn seed_bin(start_height: usize) -> f64 {
|
||||
let price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(start_height - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
cents_to_bin(price * 100.0)
|
||||
}
|
||||
|
||||
fn leading_digit(sats: u64) -> u8 {
|
||||
let log = (sats as f64).log10();
|
||||
let magnitude = 10.0_f64.powf(log.floor());
|
||||
let d = (sats as f64 / magnitude).round() as u8;
|
||||
if d >= 10 { 1 } else { d }
|
||||
}
|
||||
|
||||
fn is_round(sats: u64) -> bool {
|
||||
let log = (sats as f64).log10();
|
||||
let magnitude = 10.0_f64.powf(log.floor());
|
||||
let leading = (sats as f64 / magnitude).round();
|
||||
let round_val = leading * magnitude;
|
||||
(sats as f64 - round_val).abs() <= round_val * 0.001
|
||||
}
|
||||
|
||||
struct Stats {
|
||||
total_sq_err: f64,
|
||||
total_bias: f64,
|
||||
max_err: f64,
|
||||
total_blocks: u64,
|
||||
gt_5pct: u64,
|
||||
gt_10pct: u64,
|
||||
gt_20pct: u64,
|
||||
}
|
||||
|
||||
impl Stats {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
total_sq_err: 0.0,
|
||||
total_bias: 0.0,
|
||||
max_err: 0.0,
|
||||
total_blocks: 0,
|
||||
gt_5pct: 0,
|
||||
gt_10pct: 0,
|
||||
gt_20pct: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, err: f64) {
|
||||
self.total_sq_err += err * err;
|
||||
self.total_bias += err;
|
||||
self.total_blocks += 1;
|
||||
let abs_err = err.abs();
|
||||
if abs_err > self.max_err {
|
||||
self.max_err = abs_err;
|
||||
}
|
||||
if abs_err > BINS_5PCT {
|
||||
self.gt_5pct += 1;
|
||||
}
|
||||
if abs_err > BINS_10PCT {
|
||||
self.gt_10pct += 1;
|
||||
}
|
||||
if abs_err > BINS_20PCT {
|
||||
self.gt_20pct += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn rmse_pct(&self) -> f64 {
|
||||
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
|
||||
}
|
||||
|
||||
fn max_pct(&self) -> f64 {
|
||||
bins_to_pct(self.max_err)
|
||||
}
|
||||
|
||||
fn bias(&self) -> f64 {
|
||||
self.total_bias / self.total_blocks as f64
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let t0 = Instant::now();
|
||||
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| {
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
PathBuf::from(home).join(".brk")
|
||||
});
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("Failed to read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("Failed to parse height OHLC");
|
||||
|
||||
let height_bands: Vec<(f64, f64)> = height_ohlc
|
||||
.iter()
|
||||
.map(|ohlc| {
|
||||
let high = ohlc[1];
|
||||
let low = ohlc[2];
|
||||
if high > 0.0 && low > 0.0 {
|
||||
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Configs to compare.
|
||||
// 987654321
|
||||
let masks: &[(u16, &str)] = &[
|
||||
(0b0_0111_0111, "{1,2,3,5,6,7}"),
|
||||
(0b0_0011_0111, "{1,2,3,5,6}"),
|
||||
(0b0_0001_1111, "{1,2,3,4,5}"),
|
||||
(0b0_0001_0111, "{1,2,3,5}"),
|
||||
];
|
||||
|
||||
let start_heights: &[usize] = &[575_000, 600_000, 630_000];
|
||||
|
||||
// (mask_idx, start_idx) -> (Oracle, Stats)
|
||||
let n = masks.len() * start_heights.len();
|
||||
let mut oracles: Vec<Option<Oracle>> = (0..n).map(|_| None).collect();
|
||||
let mut stats: Vec<Stats> = (0..n).map(|_| Stats::new()).collect();
|
||||
|
||||
let idx = |m: usize, s: usize| -> usize { m * start_heights.len() + s };
|
||||
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
|
||||
let ref_config = Config::default();
|
||||
let earliest_start = *start_heights.iter().min().unwrap();
|
||||
|
||||
for h in earliest_start..total_heights {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
|
||||
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
|
||||
indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.collect_one(ft + 1)
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
} else {
|
||||
out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize()
|
||||
};
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
// Build full histogram and per-digit histograms.
|
||||
let mut full_hist = Histogram::zeros();
|
||||
let mut digit_hist: [Histogram; 9] = std::array::from_fn(|_| Histogram::zeros());
|
||||
|
||||
for (sats, output_type) in values.into_iter().zip(output_types) {
|
||||
if ref_config.excluded_output_types.contains(&output_type) {
|
||||
continue;
|
||||
}
|
||||
if *sats < ref_config.min_sats {
|
||||
continue;
|
||||
}
|
||||
if let Some(bin) = sats_to_bin(sats) {
|
||||
full_hist.increment(bin);
|
||||
if is_round(*sats) {
|
||||
let d = leading_digit(*sats);
|
||||
if (1..=9).contains(&d) {
|
||||
digit_hist[(d - 1) as usize].increment(bin);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Feed each (mask, start_height) combo.
|
||||
for (mi, &(mask, _)) in masks.iter().enumerate() {
|
||||
// Build filtered histogram for this mask.
|
||||
let mut hist = full_hist.clone();
|
||||
(0..9usize).for_each(|d| {
|
||||
if mask & (1 << d) != 0 {
|
||||
for bin in 0..NUM_BINS {
|
||||
hist[bin] -= digit_hist[d][bin];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
for (si, &sh) in start_heights.iter().enumerate() {
|
||||
if h < sh {
|
||||
continue;
|
||||
}
|
||||
let i = idx(mi, si);
|
||||
if oracles[i].is_none() {
|
||||
oracles[i] = Some(Oracle::new(
|
||||
seed_bin(sh),
|
||||
Config {
|
||||
exclude_common_round_values: false,
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
let ref_bin = oracles[i].as_mut().unwrap().process_histogram(&hist);
|
||||
|
||||
if h < height_bands.len() {
|
||||
let (high_bin, low_bin) = height_bands[h];
|
||||
if high_bin > 0.0 && low_bin > 0.0 {
|
||||
let err = if ref_bin < high_bin {
|
||||
ref_bin - high_bin
|
||||
} else if ref_bin > low_bin {
|
||||
ref_bin - low_bin
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
stats[i].update(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Print results grouped by start height.
|
||||
for (si, &sh) in start_heights.iter().enumerate() {
|
||||
println!();
|
||||
println!("@ {}k:", sh / 1000);
|
||||
println!(
|
||||
" {:<16} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
|
||||
"Digits", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
|
||||
);
|
||||
println!(" {}", "-".repeat(72));
|
||||
for (mi, &(_, label)) in masks.iter().enumerate() {
|
||||
let s = &stats[idx(mi, si)];
|
||||
println!(
|
||||
" {:<16} {:>8} {:>7.3}% {:>7.1}% {:>6} {:>6} {:>6} {:>+8.2}",
|
||||
label,
|
||||
s.total_blocks,
|
||||
s.rmse_pct(),
|
||||
s.max_pct(),
|
||||
s.gt_5pct,
|
||||
s.gt_10pct,
|
||||
s.gt_20pct,
|
||||
s.bias()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
println!("\nDone in {:.1}s", t0.elapsed().as_secs_f64());
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
//! Verify oracle determinism: oracles started from different heights converge
|
||||
//! to identical ref_bin values after the ring buffer fills.
|
||||
//! Verify the production restart property: an oracle restored via
|
||||
//! `from_checkpoint` (seeded from the previous block's stored cents price,
|
||||
//! replayed over the last `window_size` blocks) produces bit-exact `ref_bin`
|
||||
//! values matching a continuously-running oracle from the restart height
|
||||
//! onward.
|
||||
//!
|
||||
//! Creates a reference oracle at height 575k and test oracles every 1000 blocks
|
||||
//! up to 630k. After window_size blocks, each test oracle should produce the
|
||||
//! same ref_bin as the reference, proving the truncated EMA provides
|
||||
//! start-point independence.
|
||||
//! Mirrors the production transaction filter exactly, so it exercises the same code path
|
||||
//! `brk_computer::price::compute::feed_blocks` uses at runtime.
|
||||
//!
|
||||
//! Run with: cargo run -p brk_oracle --example determinism --release
|
||||
|
||||
@@ -12,26 +13,35 @@ use std::path::PathBuf;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{
|
||||
Config, Histogram, Oracle, PRICES, START_HEIGHT, cents_to_bin, default_eligible_bin,
|
||||
bin_to_cents, cents_to_bin, Config, HistogramRaw, Oracle, PaymentFilter, START_HEIGHT_FAST,
|
||||
START_HEIGHT_SLOW,
|
||||
};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
fn seed_bin(height: usize) -> f64 {
|
||||
let price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(height - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
cents_to_bin(price * 100.0)
|
||||
struct Block {
|
||||
height: usize,
|
||||
values: Vec<Sats>,
|
||||
output_types: Vec<OutputType>,
|
||||
tx_starts: Vec<usize>,
|
||||
out_start: usize,
|
||||
out_end: usize,
|
||||
}
|
||||
|
||||
struct TestRun {
|
||||
start_height: usize,
|
||||
oracle: Option<Oracle>,
|
||||
converged_at: Option<usize>,
|
||||
diverged_after: bool,
|
||||
fn build_histogram(block: &Block) -> HistogramRaw {
|
||||
let tx_outputs = (0..block.tx_starts.len()).map(|tx| {
|
||||
let lo = block.tx_starts[tx] - block.out_start;
|
||||
let hi = block
|
||||
.tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - block.out_start)
|
||||
.unwrap_or(block.out_end - block.out_start);
|
||||
block.values[lo..hi]
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(block.output_types[lo..hi].iter().copied())
|
||||
});
|
||||
PaymentFilter::for_height(block.height).histogram(tx_outputs)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
@@ -45,62 +55,54 @@ fn main() {
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
let config = Config::default();
|
||||
let window_size = config.window_size;
|
||||
let fast_config = Config::default();
|
||||
let window_size = fast_config.window_size;
|
||||
|
||||
let restart_offset = 1000;
|
||||
let end_offset = restart_offset + window_size * 4;
|
||||
let end_height = (START_HEIGHT_FAST + end_offset).min(total_heights);
|
||||
let restart_at = START_HEIGHT_FAST + restart_offset;
|
||||
let warmup_start = restart_at - window_size;
|
||||
let load_start = START_HEIGHT_SLOW;
|
||||
|
||||
assert!(
|
||||
end_height > restart_at,
|
||||
"indexer has {total_heights} blocks; need at least {} to test restart at {restart_at}",
|
||||
restart_at + 1
|
||||
);
|
||||
|
||||
println!(
|
||||
"Loading {} blocks ({load_start}..{end_height})...",
|
||||
end_height - load_start
|
||||
);
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
|
||||
|
||||
// Reference oracle at 575k.
|
||||
let ref_start = START_HEIGHT;
|
||||
let mut ref_oracle = Oracle::new(seed_bin(ref_start), Config::default());
|
||||
|
||||
// Test oracles every 1000 blocks from 576k to 630k.
|
||||
let mut runs: Vec<TestRun> = (576_000..=630_000)
|
||||
.step_by(1000)
|
||||
.map(|h| TestRun {
|
||||
start_height: h,
|
||||
oracle: None,
|
||||
converged_at: None,
|
||||
diverged_after: false,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let last_start = runs.last().map(|r| r.start_height).unwrap_or(ref_start);
|
||||
// Process enough blocks for all oracles to converge + verification margin.
|
||||
let end_height = (last_start + window_size + 100).min(total_heights);
|
||||
|
||||
for h in START_HEIGHT..end_height {
|
||||
let mut blocks: Vec<Block> = Vec::with_capacity(end_height - load_start);
|
||||
for h in load_start..end_height {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
|
||||
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
|
||||
indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.collect_one(ft + 1)
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
} else {
|
||||
out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize()
|
||||
};
|
||||
let block_first_tx = ft.to_usize() + 1;
|
||||
let tx_count = next_ft.to_usize() - block_first_tx;
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
txout_cursor.advance(block_first_tx - txout_cursor.position());
|
||||
let mut tx_starts: Vec<usize> = Vec::with_capacity(tx_count);
|
||||
for _ in 0..tx_count {
|
||||
tx_starts.push(txout_cursor.next().unwrap().to_usize());
|
||||
}
|
||||
let out_start = tx_starts.first().copied().unwrap_or(out_end);
|
||||
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
@@ -112,95 +114,85 @@ fn main() {
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
let mut hist = Histogram::zeros();
|
||||
for (sats, output_type) in values.into_iter().zip(output_types) {
|
||||
if let Some(bin) = default_eligible_bin(sats, output_type) {
|
||||
hist.increment(bin as usize);
|
||||
}
|
||||
}
|
||||
|
||||
let ref_bin = ref_oracle.process_histogram(&hist);
|
||||
|
||||
for run in &mut runs {
|
||||
if h < run.start_height {
|
||||
continue;
|
||||
}
|
||||
if run.oracle.is_none() {
|
||||
run.oracle = Some(Oracle::new(seed_bin(run.start_height), Config::default()));
|
||||
}
|
||||
let test_bin = run.oracle.as_mut().unwrap().process_histogram(&hist);
|
||||
|
||||
if run.converged_at.is_some() {
|
||||
if test_bin != ref_bin {
|
||||
run.diverged_after = true;
|
||||
}
|
||||
} else if test_bin == ref_bin {
|
||||
run.converged_at = Some(h);
|
||||
}
|
||||
}
|
||||
blocks.push(Block {
|
||||
height: h,
|
||||
values,
|
||||
output_types,
|
||||
tx_starts,
|
||||
out_start,
|
||||
out_end,
|
||||
});
|
||||
}
|
||||
|
||||
// Print results.
|
||||
println!();
|
||||
println!("{:<12} {:>16} {:>8}", "Start", "Converged at", "Blocks");
|
||||
println!("{}", "-".repeat(40));
|
||||
|
||||
let mut max_blocks = 0usize;
|
||||
let mut failed = Vec::new();
|
||||
let mut diverged = Vec::new();
|
||||
|
||||
for run in &runs {
|
||||
if let Some(converged) = run.converged_at {
|
||||
let blocks = converged - run.start_height;
|
||||
if blocks > max_blocks {
|
||||
max_blocks = blocks;
|
||||
let mut continuous = Oracle::from_seed();
|
||||
let continuous_bins: Vec<f64> = blocks
|
||||
.iter()
|
||||
.map(|b| {
|
||||
if b.height == START_HEIGHT_FAST {
|
||||
continuous.reconfigure(fast_config);
|
||||
}
|
||||
println!("{:<12} {:>16} {:>8}", run.start_height, converged, blocks);
|
||||
if run.diverged_after {
|
||||
diverged.push(run.start_height);
|
||||
}
|
||||
} else {
|
||||
println!("{:<12} {:>16} {:>8}", run.start_height, "NEVER", "-");
|
||||
failed.push(run.start_height);
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
continuous.process_histogram(&build_histogram(b))
|
||||
})
|
||||
.collect();
|
||||
println!(
|
||||
"{}/{} converged, max {} blocks to converge (window_size={})",
|
||||
runs.len() - failed.len(),
|
||||
runs.len(),
|
||||
max_blocks,
|
||||
window_size,
|
||||
"Continuous oracle: {} blocks processed",
|
||||
continuous_bins.len()
|
||||
);
|
||||
|
||||
if !diverged.is_empty() {
|
||||
println!("DIVERGED after convergence: {:?}", diverged);
|
||||
}
|
||||
if !failed.is_empty() {
|
||||
println!("NEVER converged: {:?}", failed);
|
||||
let prev_bin = continuous_bins[restart_at - load_start - 1];
|
||||
let seed_bin = cents_to_bin(bin_to_cents(prev_bin) as f64);
|
||||
println!(
|
||||
"Restart at {restart_at}: prev_bin={prev_bin:.4} -> cents -> seed_bin={seed_bin:.4} (delta {:.6})",
|
||||
seed_bin - prev_bin
|
||||
);
|
||||
|
||||
let warmup_slice = &blocks[warmup_start - load_start..restart_at - load_start];
|
||||
let mut restored = Oracle::from_checkpoint(seed_bin, fast_config, |o| {
|
||||
for b in warmup_slice {
|
||||
o.process_histogram(&build_histogram(b));
|
||||
}
|
||||
});
|
||||
|
||||
let restored_bins: Vec<f64> = blocks[restart_at - load_start..]
|
||||
.iter()
|
||||
.map(|b| restored.process_histogram(&build_histogram(b)))
|
||||
.collect();
|
||||
println!("Restored oracle: {} blocks processed", restored_bins.len());
|
||||
|
||||
let mut mismatches: Vec<(usize, f64, f64)> = Vec::new();
|
||||
for (i, &r) in restored_bins.iter().enumerate() {
|
||||
let c = continuous_bins[restart_at - load_start + i];
|
||||
if r != c {
|
||||
mismatches.push((restart_at + i, c, r));
|
||||
}
|
||||
}
|
||||
|
||||
// Assertions.
|
||||
assert!(
|
||||
failed.is_empty(),
|
||||
"{} oracles never converged: {:?}",
|
||||
failed.len(),
|
||||
failed
|
||||
);
|
||||
assert!(
|
||||
diverged.is_empty(),
|
||||
"{} oracles diverged after convergence: {:?}",
|
||||
diverged.len(),
|
||||
diverged
|
||||
);
|
||||
assert!(
|
||||
max_blocks <= window_size * 2,
|
||||
"Convergence took {} blocks, expected <= {} (2 * window_size)",
|
||||
max_blocks,
|
||||
window_size * 2
|
||||
println!();
|
||||
if mismatches.is_empty() {
|
||||
println!(
|
||||
"All {} blocks from {restart_at} onward match exactly.",
|
||||
restored_bins.len()
|
||||
);
|
||||
} else {
|
||||
println!(
|
||||
"{} of {} blocks differ (showing up to 5):",
|
||||
mismatches.len(),
|
||||
restored_bins.len()
|
||||
);
|
||||
for (h, c, r) in mismatches.iter().take(5) {
|
||||
println!(
|
||||
" h={h}: continuous={c:.6}, restored={r:.6}, delta={:.6}",
|
||||
r - c
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
mismatches.len(),
|
||||
0,
|
||||
"restored oracle diverged from continuous oracle"
|
||||
);
|
||||
|
||||
println!();
|
||||
println!("All assertions passed!");
|
||||
println!("Assertion passed: from_checkpoint restart is bit-exact.");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
//! Dump the RAW per-output data over a height range for fully offline analysis.
|
||||
//! Nothing is filtered or binned, so any downstream filter (round-BTC tolerance,
|
||||
//! dust floor, type exclusion, OP_RETURN / batch-payout tx drops, log-bin
|
||||
//! resolution) can be reconstructed in analysis WITHOUT re-dumping.
|
||||
//!
|
||||
//! For every non-coinbase output in [ORACLE_START, ORACLE_END) (default
|
||||
//! 500000..510000) one row is written:
|
||||
//! oracle_outputs_{start}_{end}.csv height,tx,sats,otype
|
||||
//! where `tx` is the 0-based index of the (non-coinbase) transaction within the
|
||||
//! block (so OP_RETURN-tx and >N-output-tx drops can be reapplied by grouping),
|
||||
//! `sats` is the exact output value, and `otype` is `OutputType as u8`.
|
||||
//!
|
||||
//! Plus per-block metadata:
|
||||
//! oracle_meta_{start}_{end}.csv height,timestamp,ex_low,ex_high,ex_close
|
||||
//!
|
||||
//! Run: cargo run -p brk_oracle --example dump_hist --release
|
||||
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{BufWriter, Write},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
fn main() {
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from(std::env::var("HOME").unwrap()).join(".brk"));
|
||||
let out_dir = std::env::var("DUMP_DIR").unwrap_or_else(|_| "/tmp".to_string());
|
||||
let start: usize = std::env::var("ORACLE_START")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(500_000);
|
||||
let end: usize = std::env::var("ORACLE_END")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(510_000);
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
let end = end.min(total_heights);
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("parse height OHLC");
|
||||
|
||||
let timestamps: Vec<brk_types::Timestamp> = indexer.vecs.blocks.timestamp.collect();
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
|
||||
let mut tx_starts: Vec<usize> = Vec::new();
|
||||
|
||||
let out_path = format!("{out_dir}/oracle_outputs_{start}_{end}.csv");
|
||||
let meta_path = format!("{out_dir}/oracle_meta_{start}_{end}.csv");
|
||||
let mut out_w = BufWriter::new(File::create(&out_path).expect("create outputs csv"));
|
||||
let mut meta_w = BufWriter::new(File::create(&meta_path).expect("create meta csv"));
|
||||
writeln!(out_w, "height,tx,sats,otype").unwrap();
|
||||
writeln!(meta_w, "height,timestamp,ex_low,ex_high,ex_close").unwrap();
|
||||
|
||||
eprintln!(
|
||||
"otype legend: OpReturn={} P2TR={}",
|
||||
OutputType::OpReturn as u8,
|
||||
OutputType::P2TR as u8
|
||||
);
|
||||
|
||||
let mut rows: u64 = 0;
|
||||
for h in start..end {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
let block_first_tx = ft.to_usize() + 1; // skip coinbase
|
||||
let tx_count = next_ft.to_usize() - block_first_tx;
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
txout_cursor.advance(block_first_tx - txout_cursor.position());
|
||||
tx_starts.clear();
|
||||
for _ in 0..tx_count {
|
||||
tx_starts.push(txout_cursor.next().unwrap().to_usize());
|
||||
}
|
||||
let out_start = tx_starts.first().copied().unwrap_or(out_end);
|
||||
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
for tx in 0..tx_count {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
for i in lo..hi {
|
||||
writeln!(out_w, "{h},{tx},{},{}", *values[i], output_types[i] as u8).unwrap();
|
||||
rows += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let o = height_ohlc.get(h).copied().unwrap_or([0.0; 4]);
|
||||
writeln!(
|
||||
meta_w,
|
||||
"{h},{},{:.2},{:.2},{:.2}",
|
||||
*timestamps[h], o[2], o[1], o[3]
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
out_w.flush().unwrap();
|
||||
meta_w.flush().unwrap();
|
||||
eprintln!("wrote {out_path} ({rows} output rows)");
|
||||
eprintln!("wrote {meta_path}");
|
||||
}
|
||||
@@ -0,0 +1,690 @@
|
||||
//! Compare oracle filter/EMA variants against the historical OHLC set.
|
||||
//!
|
||||
//! This is a diagnostic harness, not production code. It mirrors the production
|
||||
//! state machine closely enough to compare candidate changes in one pass over
|
||||
//! the indexed chain while recording the recent bad-lock heights.
|
||||
//!
|
||||
//! Run:
|
||||
//! cargo run -p brk_oracle --example experiment --release
|
||||
|
||||
use std::{cmp::Ordering, env, path::PathBuf};
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{
|
||||
bin_to_cents, cents_to_bin, seed_bin as oracle_seed_bin, Config, PaymentFilter,
|
||||
BINS_PER_DECADE, NUM_BINS, START_HEIGHT_FAST, START_HEIGHT_SLOW,
|
||||
};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
const GENESIS_DAY: u32 = 14252;
|
||||
const BINS_5PCT: f64 = 4.24;
|
||||
const BINS_10PCT: f64 = 8.28;
|
||||
const BINS_20PCT: f64 = 15.84;
|
||||
const STENCIL_OFFSETS: [i32; 19] = [
|
||||
-400, -340, -305, -260, -200, -165, -140, -120, -105, -60, 0, 35, 60, 95, 140, 200, 260, 340,
|
||||
400,
|
||||
];
|
||||
const N_ARMS: usize = STENCIL_OFFSETS.len();
|
||||
const TARGET_HEIGHTS: &[usize] = &[952_286, 952_287, 952_288, 952_289, 952_290];
|
||||
|
||||
fn bins_to_pct(bins: f64) -> f64 {
|
||||
(10.0_f64.powf(bins / BINS_PER_DECADE as f64) - 1.0) * 100.0
|
||||
}
|
||||
|
||||
fn timestamp_to_year(ts: u32) -> u16 {
|
||||
let years_since_1970 = ts as f64 / 31_557_600.0;
|
||||
(1970.0 + years_since_1970) as u16
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ShapeAnchor {
|
||||
weight: f64,
|
||||
profile: [f64; N_ARMS],
|
||||
}
|
||||
|
||||
impl ShapeAnchor {
|
||||
fn new(weight: f64) -> Self {
|
||||
Self {
|
||||
weight,
|
||||
profile: [1.0 / N_ARMS as f64; N_ARMS],
|
||||
}
|
||||
}
|
||||
|
||||
fn score(&self, state: &OracleState, center: i64) -> f64 {
|
||||
if self.weight == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
self.weight
|
||||
* normalized_arms_at(state, center)
|
||||
.map(|arms| {
|
||||
1.0 - (0..N_ARMS)
|
||||
.map(|i| (arms[i] - self.profile[i]).abs())
|
||||
.sum::<f64>()
|
||||
})
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
fn update(&mut self, state: &OracleState, pick: i64) {
|
||||
const BETA: f64 = 0.004;
|
||||
if self.weight == 0.0 {
|
||||
return;
|
||||
}
|
||||
if let Some(arms) = normalized_arms_at(state, pick) {
|
||||
for (p, arm) in self.profile.iter_mut().zip(arms) {
|
||||
*p = (1.0 - BETA) * *p + BETA * arm;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct OracleState {
|
||||
config: Config,
|
||||
ring: Vec<Vec<f64>>,
|
||||
nonzero: Vec<Vec<usize>>,
|
||||
weights: Vec<f64>,
|
||||
cursor: usize,
|
||||
filled: usize,
|
||||
ref_bin: f64,
|
||||
warmup: bool,
|
||||
shape: ShapeAnchor,
|
||||
}
|
||||
|
||||
impl OracleState {
|
||||
fn new(ref_bin: f64, config: Config) -> Self {
|
||||
let weights = weights(config.window_size, config.alpha);
|
||||
Self {
|
||||
ring: vec![vec![0.0; NUM_BINS]; config.window_size],
|
||||
nonzero: vec![Vec::new(); config.window_size],
|
||||
weights,
|
||||
cursor: 0,
|
||||
filled: 0,
|
||||
ref_bin,
|
||||
warmup: false,
|
||||
shape: ShapeAnchor::new(config.shape_weight),
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
fn reconfigure(&mut self, config: Config) {
|
||||
let kept = self.recent(config.window_size);
|
||||
let mut next = Self::new(self.ref_bin, config);
|
||||
next.warmup = true;
|
||||
for hist in kept {
|
||||
next.push_existing(hist);
|
||||
}
|
||||
next.warmup = false;
|
||||
*self = next;
|
||||
}
|
||||
|
||||
fn recent(&self, n: usize) -> Vec<Vec<f64>> {
|
||||
(0..self.filled.min(n))
|
||||
.rev()
|
||||
.map(|age| self.ring[self.index_at_age(age)].clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn index_at_age(&self, age: usize) -> usize {
|
||||
(self.cursor + self.ring.len() - 1 - age) % self.ring.len()
|
||||
}
|
||||
|
||||
fn start_block(&mut self) {
|
||||
for bin in self.nonzero[self.cursor].drain(..) {
|
||||
self.ring[self.cursor][bin] = 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
fn add(&mut self, bin: usize, weight: f64) {
|
||||
let slot = &mut self.ring[self.cursor];
|
||||
if slot[bin] == 0.0 {
|
||||
self.nonzero[self.cursor].push(bin);
|
||||
}
|
||||
slot[bin] += weight;
|
||||
}
|
||||
|
||||
fn push_existing(&mut self, hist: Vec<f64>) {
|
||||
self.start_block();
|
||||
for (bin, value) in hist
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter(|(_, value)| *value != 0.0)
|
||||
{
|
||||
self.add(bin, value);
|
||||
}
|
||||
self.finish_block();
|
||||
}
|
||||
|
||||
fn finish_block(&mut self) {
|
||||
self.cursor = (self.cursor + 1) % self.ring.len();
|
||||
self.filled = (self.filled + 1).min(self.ring.len());
|
||||
if self.warmup {
|
||||
return;
|
||||
}
|
||||
self.ref_bin = find_best_bin(self);
|
||||
let mut shape = self.shape.clone();
|
||||
shape.update(self, self.ref_bin.round() as i64);
|
||||
self.shape = shape;
|
||||
}
|
||||
|
||||
fn value_at(&self, bin: i64) -> f64 {
|
||||
if bin < 0 || bin as usize >= NUM_BINS {
|
||||
return 0.0;
|
||||
}
|
||||
let bin = bin as usize;
|
||||
(0..self.filled)
|
||||
.map(|age| self.weights[age] * self.ring[self.index_at_age(age)][bin])
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
|
||||
fn weights(window_size: usize, alpha: f64) -> Vec<f64> {
|
||||
let decay = 1.0 - alpha;
|
||||
(0..window_size)
|
||||
.map(|i| alpha * decay.powi(i as i32))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn normalized_arms_at(state: &OracleState, center: i64) -> Option<[f64; N_ARMS]> {
|
||||
let mut arms = STENCIL_OFFSETS.map(|offset| state.value_at(center + offset as i64));
|
||||
let sum: f64 = arms.iter().sum();
|
||||
if sum <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
for arm in &mut arms {
|
||||
*arm /= sum;
|
||||
}
|
||||
Some(arms)
|
||||
}
|
||||
|
||||
fn find_best_bin(state: &OracleState) -> f64 {
|
||||
let center = state.ref_bin.round() as usize;
|
||||
let search_start = center.saturating_sub(state.config.search_below);
|
||||
let search_end = (center + state.config.search_above + 1).min(NUM_BINS);
|
||||
if search_start >= search_end {
|
||||
return state.ref_bin;
|
||||
}
|
||||
|
||||
let mut arm_peaks = [0.0f64; N_ARMS];
|
||||
for (i, &offset) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
for bin in search_start..search_end {
|
||||
arm_peaks[i] = arm_peaks[i].max(state.value_at(bin as i64 + offset as i64));
|
||||
}
|
||||
}
|
||||
|
||||
let score = |bin: usize| -> f64 {
|
||||
let mut total = 0.0;
|
||||
for (i, &offset) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
if arm_peaks[i] > 0.0 {
|
||||
total += state.value_at(bin as i64 + offset as i64) / arm_peaks[i];
|
||||
}
|
||||
}
|
||||
total + state.shape.score(state, bin as i64)
|
||||
};
|
||||
|
||||
let mut best_bin = search_start;
|
||||
let mut best_score = score(search_start);
|
||||
for bin in (search_start + 1)..search_end {
|
||||
let candidate = score(bin);
|
||||
if candidate > best_score {
|
||||
best_score = candidate;
|
||||
best_bin = bin;
|
||||
}
|
||||
}
|
||||
|
||||
let score_center = best_score;
|
||||
let score_left = if best_bin > search_start {
|
||||
score(best_bin - 1)
|
||||
} else {
|
||||
score_center
|
||||
};
|
||||
let score_right = if best_bin + 1 < search_end {
|
||||
score(best_bin + 1)
|
||||
} else {
|
||||
score_center
|
||||
};
|
||||
let denom = score_left - 2.0 * score_center + score_right;
|
||||
let sub_bin = if denom.abs() > 1e-10 {
|
||||
(0.5 * (score_left - score_right) / denom).clamp(-0.5, 0.5)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
best_bin as f64 + sub_bin
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct VariantCfg {
|
||||
name: String,
|
||||
fast_alpha: f64,
|
||||
fast_window: usize,
|
||||
max_outputs: Option<usize>,
|
||||
max_outputs_until: usize,
|
||||
max_outputs_after: Option<usize>,
|
||||
}
|
||||
|
||||
struct Variant {
|
||||
cfg: VariantCfg,
|
||||
state: OracleState,
|
||||
overall: YearStats,
|
||||
years: Vec<YearStats>,
|
||||
bias: f64,
|
||||
target_prices: Vec<(usize, f64)>,
|
||||
}
|
||||
|
||||
fn cap_label(cap: Option<usize>) -> String {
|
||||
cap.map(|cap| cap.to_string())
|
||||
.unwrap_or_else(|| "none".to_string())
|
||||
}
|
||||
|
||||
fn target_price(target_prices: &[(usize, f64)], height: usize) -> Option<f64> {
|
||||
target_prices
|
||||
.iter()
|
||||
.find(|(h, _)| *h == height)
|
||||
.map(|(_, price)| *price)
|
||||
}
|
||||
|
||||
fn fixes_bad_lock(target_prices: &[(usize, f64)]) -> bool {
|
||||
target_price(target_prices, 952_287).is_some_and(|price| price > 62_000.0)
|
||||
&& target_price(target_prices, 952_288).is_some_and(|price| price > 62_000.0)
|
||||
}
|
||||
|
||||
impl Variant {
|
||||
fn new(cfg: VariantCfg, seed_bin: f64) -> Self {
|
||||
Self {
|
||||
cfg,
|
||||
state: OracleState::new(seed_bin, Config::slow()),
|
||||
overall: YearStats::new(0),
|
||||
years: Vec::new(),
|
||||
bias: 0.0,
|
||||
target_prices: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn fast_config(&self) -> Config {
|
||||
Config {
|
||||
alpha: self.cfg.fast_alpha,
|
||||
window_size: self.cfg.fast_window,
|
||||
..Config::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_reconfigure(&mut self, height: usize) {
|
||||
if height == START_HEIGHT_FAST {
|
||||
self.state.reconfigure(self.fast_config());
|
||||
}
|
||||
}
|
||||
|
||||
fn should_drop_tx(&self, height: usize, output_count: usize) -> bool {
|
||||
if height < self.cfg.max_outputs_until {
|
||||
self.cfg.max_outputs.is_some_and(|max| output_count > max)
|
||||
} else {
|
||||
self.cfg
|
||||
.max_outputs_after
|
||||
.is_some_and(|max| output_count > max)
|
||||
}
|
||||
}
|
||||
|
||||
fn add_tx(&mut self, bins: &[u16], height: usize, output_count: usize) {
|
||||
if bins.is_empty() || self.should_drop_tx(height, output_count) {
|
||||
return;
|
||||
}
|
||||
for &bin in bins {
|
||||
self.state.add(bin as usize, 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_block(&mut self, height: usize) {
|
||||
self.state.finish_block();
|
||||
if TARGET_HEIGHTS.contains(&height) {
|
||||
self.target_prices
|
||||
.push((height, bin_to_cents(self.state.ref_bin) as f64 / 100.0));
|
||||
}
|
||||
}
|
||||
|
||||
fn update_stats(
|
||||
&mut self,
|
||||
height: usize,
|
||||
height_bands: &[(f64, f64)],
|
||||
height_ohlc: &[[f64; 4]],
|
||||
height_years: &[u16],
|
||||
) {
|
||||
if height >= height_bands.len() {
|
||||
return;
|
||||
}
|
||||
let (high_bin, low_bin) = height_bands[height];
|
||||
if high_bin <= 0.0 || low_bin <= 0.0 {
|
||||
return;
|
||||
}
|
||||
let err = if self.state.ref_bin < high_bin {
|
||||
self.state.ref_bin - high_bin
|
||||
} else if self.state.ref_bin > low_bin {
|
||||
self.state.ref_bin - low_bin
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let exchange_high = height_ohlc[height][1];
|
||||
let exchange_low = height_ohlc[height][2];
|
||||
self.overall.update(err, exchange_high, exchange_low);
|
||||
self.bias += err;
|
||||
|
||||
let year = height_years[height];
|
||||
if self.years.last().is_none_or(|stats| stats.year != year) {
|
||||
self.years.push(YearStats::new(year));
|
||||
}
|
||||
self.years
|
||||
.last_mut()
|
||||
.unwrap()
|
||||
.update(err, exchange_high, exchange_low);
|
||||
}
|
||||
}
|
||||
|
||||
struct YearStats {
|
||||
year: u16,
|
||||
total_sq_err: f64,
|
||||
max_err: f64,
|
||||
total_blocks: u64,
|
||||
gt_5pct: u64,
|
||||
gt_10pct: u64,
|
||||
gt_20pct: u64,
|
||||
errors: Vec<f64>,
|
||||
}
|
||||
|
||||
impl YearStats {
|
||||
fn new(year: u16) -> Self {
|
||||
Self {
|
||||
year,
|
||||
total_sq_err: 0.0,
|
||||
max_err: 0.0,
|
||||
total_blocks: 0,
|
||||
gt_5pct: 0,
|
||||
gt_10pct: 0,
|
||||
gt_20pct: 0,
|
||||
errors: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, err: f64, _exchange_high: f64, _exchange_low: f64) {
|
||||
let abs_err = err.abs();
|
||||
self.total_sq_err += err * err;
|
||||
self.total_blocks += 1;
|
||||
self.errors.push(bins_to_pct(abs_err));
|
||||
self.max_err = self.max_err.max(abs_err);
|
||||
if abs_err > BINS_5PCT {
|
||||
self.gt_5pct += 1;
|
||||
}
|
||||
if abs_err > BINS_10PCT {
|
||||
self.gt_10pct += 1;
|
||||
}
|
||||
if abs_err > BINS_20PCT {
|
||||
self.gt_20pct += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn rmse_pct(&self) -> f64 {
|
||||
if self.total_blocks == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
|
||||
}
|
||||
|
||||
fn max_pct(&self) -> f64 {
|
||||
bins_to_pct(self.max_err)
|
||||
}
|
||||
|
||||
fn percentile(&self, p: f64) -> f64 {
|
||||
if self.errors.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut errors = self.errors.clone();
|
||||
errors.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
|
||||
let idx = ((p / 100.0) * (errors.len() - 1) as f64).round() as usize;
|
||||
errors[idx.min(errors.len() - 1)]
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from(std::env::var("HOME").unwrap()).join(".brk"));
|
||||
let end_override = std::env::var("ORACLE_END")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<usize>().ok());
|
||||
let stats_start = std::env::var("ORACLE_STATS_START")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<usize>().ok())
|
||||
.unwrap_or(START_HEIGHT_SLOW)
|
||||
.max(START_HEIGHT_SLOW);
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
let end = end_override.unwrap_or(total_heights).min(total_heights);
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("parse height OHLC");
|
||||
let height_bands: Vec<(f64, f64)> = height_ohlc
|
||||
.iter()
|
||||
.map(|ohlc| {
|
||||
let high = ohlc[1];
|
||||
let low = ohlc[2];
|
||||
if high > 0.0 && low > 0.0 {
|
||||
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let timestamps: Vec<brk_types::Timestamp> = indexer.vecs.blocks.timestamp.collect();
|
||||
let height_years: Vec<u16> = timestamps
|
||||
.iter()
|
||||
.map(|ts| timestamp_to_year(**ts))
|
||||
.collect();
|
||||
let _height_day1s: Vec<usize> = timestamps
|
||||
.iter()
|
||||
.map(|ts| (**ts / 86_400).saturating_sub(GENESIS_DAY) as usize)
|
||||
.collect();
|
||||
|
||||
let seed_bin = oracle_seed_bin();
|
||||
|
||||
let current_alpha = 2.0 / 7.0;
|
||||
let current_window = 12;
|
||||
let mut cfgs = Vec::<VariantCfg>::new();
|
||||
let mut add_cfg = |name: String,
|
||||
max_outputs: Option<usize>,
|
||||
max_outputs_until: usize,
|
||||
max_outputs_after: Option<usize>| {
|
||||
cfgs.push(VariantCfg {
|
||||
name,
|
||||
fast_alpha: current_alpha,
|
||||
fast_window: current_window,
|
||||
max_outputs,
|
||||
max_outputs_until,
|
||||
max_outputs_after,
|
||||
});
|
||||
};
|
||||
|
||||
for post in [200, 250] {
|
||||
add_cfg(
|
||||
format!("pre100_post{post}"),
|
||||
Some(100),
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT,
|
||||
Some(post),
|
||||
);
|
||||
}
|
||||
|
||||
cfgs.dedup_by(|a, b| a.name == b.name);
|
||||
if let Ok(only) = env::var("BRK_ORACLE_EXPERIMENT_ONLY") {
|
||||
let names = only
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.collect::<Vec<_>>();
|
||||
cfgs.retain(|cfg| names.iter().any(|name| *name == cfg.name));
|
||||
}
|
||||
let mut variants: Vec<Variant> = cfgs
|
||||
.into_iter()
|
||||
.map(|cfg| Variant::new(cfg, seed_bin))
|
||||
.collect();
|
||||
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
|
||||
let mut tx_starts: Vec<usize> = Vec::new();
|
||||
let mut values: Vec<Sats> = Vec::new();
|
||||
let mut output_types: Vec<OutputType> = Vec::new();
|
||||
let mut bins: Vec<u16> = Vec::new();
|
||||
|
||||
eprintln!(
|
||||
"running {} variants over heights {START_HEIGHT_SLOW}..{end}; stats from {stats_start}",
|
||||
variants.len()
|
||||
);
|
||||
|
||||
for h in START_HEIGHT_SLOW..end {
|
||||
if h % 25_000 == 0 {
|
||||
eprintln!("height {h}");
|
||||
}
|
||||
for variant in &mut variants {
|
||||
variant.maybe_reconfigure(h);
|
||||
variant.state.start_block();
|
||||
}
|
||||
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
let block_first_tx = ft.to_usize() + 1;
|
||||
let tx_count = next_ft.to_usize() - block_first_tx;
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
txout_cursor.advance(block_first_tx - txout_cursor.position());
|
||||
tx_starts.clear();
|
||||
for _ in 0..tx_count {
|
||||
tx_starts.push(txout_cursor.next().unwrap().to_usize());
|
||||
}
|
||||
let out_start = tx_starts.first().copied().unwrap_or(out_end);
|
||||
|
||||
indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_into_at(out_start, out_end, &mut values);
|
||||
indexer.vecs.outputs.output_type.collect_range_into_at(
|
||||
out_start,
|
||||
out_end,
|
||||
&mut output_types,
|
||||
);
|
||||
|
||||
for tx in 0..tx_count {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
if output_types[lo..hi].contains(&OutputType::OpReturn) {
|
||||
continue;
|
||||
}
|
||||
bins.clear();
|
||||
for i in lo..hi {
|
||||
if let Some(bin) = PaymentFilter::eligible_bin(values[i], output_types[i]) {
|
||||
bins.push(bin);
|
||||
}
|
||||
}
|
||||
for variant in &mut variants {
|
||||
variant.add_tx(&bins, h, hi - lo);
|
||||
}
|
||||
}
|
||||
|
||||
for variant in &mut variants {
|
||||
variant.finish_block(h);
|
||||
if h >= stats_start {
|
||||
variant.update_stats(h, &height_bands, &height_ohlc, &height_years);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
variants.sort_by(|a, b| {
|
||||
fixes_bad_lock(&b.target_prices)
|
||||
.cmp(&fixes_bad_lock(&a.target_prices))
|
||||
.then_with(|| {
|
||||
a.overall
|
||||
.rmse_pct()
|
||||
.partial_cmp(&b.overall.rmse_pct())
|
||||
.unwrap_or(Ordering::Equal)
|
||||
})
|
||||
.then_with(|| a.overall.gt_5pct.cmp(&b.overall.gt_5pct))
|
||||
});
|
||||
|
||||
println!(
|
||||
"variant\tpre_cap\tpost_cap\tfixed\tmedian\tp95\tp99\tp999\trmse\tmax\tbias_bins\tgt5\tgt10\tgt20\tp952287\tp952288\ttarget_prices\trmse_by_year\tgt5_by_year"
|
||||
);
|
||||
for variant in &variants {
|
||||
let overall = &variant.overall;
|
||||
let bias = if overall.total_blocks > 0 {
|
||||
variant.bias / overall.total_blocks as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let rmse_by_year = (2015..=2026)
|
||||
.map(|year| {
|
||||
let rmse = variant
|
||||
.years
|
||||
.iter()
|
||||
.find(|stats| stats.year == year)
|
||||
.map(YearStats::rmse_pct)
|
||||
.unwrap_or(0.0);
|
||||
format!("{year}:{rmse:.3}")
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let gt5_by_year = (2015..=2026)
|
||||
.map(|year| {
|
||||
let gt5 = variant
|
||||
.years
|
||||
.iter()
|
||||
.find(|stats| stats.year == year)
|
||||
.map(|stats| stats.gt_5pct)
|
||||
.unwrap_or(0);
|
||||
format!("{year}:{gt5}")
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
println!(
|
||||
"{}\t{}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{}\t{}\t{}\t{:.2}\t{:.2}\t{}\t{}\t{}",
|
||||
variant.cfg.name,
|
||||
cap_label(variant.cfg.max_outputs),
|
||||
cap_label(variant.cfg.max_outputs_after),
|
||||
fixes_bad_lock(&variant.target_prices),
|
||||
overall.percentile(50.0),
|
||||
overall.percentile(95.0),
|
||||
overall.percentile(99.0),
|
||||
overall.percentile(99.9),
|
||||
overall.rmse_pct(),
|
||||
overall.max_pct(),
|
||||
bias,
|
||||
overall.gt_5pct,
|
||||
overall.gt_10pct,
|
||||
overall.gt_20pct,
|
||||
target_price(&variant.target_prices, 952_287).unwrap_or(0.0),
|
||||
target_price(&variant.target_prices, 952_288).unwrap_or(0.0),
|
||||
variant
|
||||
.target_prices
|
||||
.iter()
|
||||
.map(|(height, price)| format!("{height}:{price:.2}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
rmse_by_year,
|
||||
gt5_by_year
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,272 +0,0 @@
|
||||
//! Diagnostic: sweep oracle start heights and clamp-top-N strategies.
|
||||
//!
|
||||
//! Run with: cargo run -p brk_oracle --example noise --release
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, default_eligible_bin};
|
||||
use brk_types::{Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
const BINS_5PCT: f64 = 4.24;
|
||||
const BINS_10PCT: f64 = 8.28;
|
||||
const BINS_20PCT: f64 = 15.84;
|
||||
const BPD: f64 = 200.0;
|
||||
|
||||
fn bins_to_pct(bins: f64) -> f64 {
|
||||
(10.0_f64.powf(bins / BPD) - 1.0) * 100.0
|
||||
}
|
||||
|
||||
fn seed_bin(start_height: usize) -> f64 {
|
||||
let price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(start_height - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
cents_to_bin(price * 100.0)
|
||||
}
|
||||
|
||||
/// Clamp the top N bins in `src` down to the (N+1)th highest value, writing into `dst`.
|
||||
fn clamp_top_n(src: &Histogram, dst: &mut Histogram, n: usize) {
|
||||
let mut top: Vec<u32> = src.iter().copied().filter(|&v| v > 0).collect();
|
||||
top.sort_unstable_by(|a, b| b.cmp(a));
|
||||
let clamp_to = if top.len() > n { top[n] } else { 0 };
|
||||
|
||||
for (i, &v) in src.iter().enumerate() {
|
||||
dst[i] = v.min(clamp_to.max(v.min(clamp_to)));
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let t0 = Instant::now();
|
||||
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| {
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
PathBuf::from(home).join(".brk")
|
||||
});
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("Failed to read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("Failed to parse height OHLC");
|
||||
|
||||
let height_bands: Vec<(f64, f64)> = height_ohlc
|
||||
.iter()
|
||||
.map(|ohlc| {
|
||||
let high = ohlc[1];
|
||||
let low = ohlc[2];
|
||||
if high > 0.0 && low > 0.0 {
|
||||
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Start heights: 630k, 600k, 575k, then 570k down to 500k by 5k.
|
||||
let mut start_heights: Vec<usize> = vec![630_000, 600_000, 575_000];
|
||||
let mut h = 570_000;
|
||||
while h >= 500_000 {
|
||||
start_heights.push(h);
|
||||
h -= 5_000;
|
||||
}
|
||||
let lowest = *start_heights.iter().min().unwrap();
|
||||
|
||||
// Clamp-top-N values to test: 0 (no clamp), 2, 3, 5, 10.
|
||||
let clamp_values: Vec<usize> = vec![0, 2, 3, 5, 10];
|
||||
|
||||
// Build per-block RAW histograms from the lowest start height.
|
||||
eprintln!("Building histograms from height {}...", lowest);
|
||||
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
|
||||
let first_txout_index_reader = indexer.vecs.transactions.first_txout_index.reader();
|
||||
let value_reader = indexer.vecs.outputs.value.reader();
|
||||
let output_type_reader = indexer.vecs.outputs.output_type.reader();
|
||||
|
||||
let config = Config::default();
|
||||
let total_blocks = total_heights - lowest;
|
||||
|
||||
struct BlockData {
|
||||
hist: Histogram,
|
||||
high_bin: f64,
|
||||
low_bin: f64,
|
||||
}
|
||||
|
||||
let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
|
||||
|
||||
for h in lowest..total_heights {
|
||||
let first_tx_index: TxIndex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_tx_index
|
||||
.collect_one_at(h)
|
||||
.unwrap();
|
||||
let next_first_tx_index: TxIndex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_tx_index
|
||||
.collect_one_at(h + 1)
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
|
||||
let out_start = if first_tx_index.to_usize() + 1 < next_first_tx_index.to_usize() {
|
||||
first_txout_index_reader
|
||||
.get(first_tx_index.to_usize() + 1)
|
||||
.to_usize()
|
||||
} else {
|
||||
indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.first_txout_index
|
||||
.collect_one_at(h + 1)
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize()
|
||||
};
|
||||
let out_end: usize = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.first_txout_index
|
||||
.collect_one_at(h + 1)
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
let mut hist = Histogram::zeros();
|
||||
for i in out_start..out_end {
|
||||
let sats: Sats = value_reader.get(i);
|
||||
let output_type = output_type_reader.get(i);
|
||||
if let Some(bin) = default_eligible_bin(sats, output_type) {
|
||||
hist.increment(bin as usize);
|
||||
}
|
||||
}
|
||||
|
||||
let (high_bin, low_bin) = if h < height_bands.len() {
|
||||
height_bands[h]
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
};
|
||||
|
||||
blocks.push(BlockData {
|
||||
hist,
|
||||
high_bin,
|
||||
low_bin,
|
||||
});
|
||||
|
||||
if (h - lowest).is_multiple_of(50_000) {
|
||||
eprint!(
|
||||
"\r {}/{} ({:.0}%)",
|
||||
h - lowest,
|
||||
total_blocks,
|
||||
(h - lowest) as f64 / total_blocks as f64 * 100.0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!(
|
||||
"\r {} blocks built in {:.1}s",
|
||||
blocks.len(),
|
||||
t0.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
// For each clamp value, run all start heights.
|
||||
for &clamp_n in &clamp_values {
|
||||
println!();
|
||||
let label = if clamp_n == 0 {
|
||||
"no clamp".to_string()
|
||||
} else {
|
||||
format!("clamp top {}", clamp_n)
|
||||
};
|
||||
println!("=== {} ===", label);
|
||||
println!(
|
||||
"{:>8} {:>8} {:>8} {:>8} {:>6} {:>6} {:>6} {:>8}",
|
||||
"Start", "Blocks", "RMSE%", "Worst%", ">5%", ">10%", ">20%", "Worst@"
|
||||
);
|
||||
println!("{}", "-".repeat(72));
|
||||
|
||||
for &start_height in &start_heights {
|
||||
let mut oracle = Oracle::new(seed_bin(start_height), config.clone());
|
||||
let block_offset = start_height - lowest;
|
||||
|
||||
let mut worst_err: f64 = 0.0;
|
||||
let mut worst_height: usize = 0;
|
||||
let mut gt_5: u64 = 0;
|
||||
let mut gt_10: u64 = 0;
|
||||
let mut gt_20: u64 = 0;
|
||||
let mut total_sq_err: f64 = 0.0;
|
||||
let mut total_measured: u64 = 0;
|
||||
|
||||
let mut clamped_hist = Histogram::zeros();
|
||||
for (i, bd) in blocks[block_offset..].iter().enumerate() {
|
||||
if clamp_n > 0 {
|
||||
clamp_top_n(&bd.hist, &mut clamped_hist, clamp_n);
|
||||
oracle.process_histogram(&clamped_hist);
|
||||
} else {
|
||||
oracle.process_histogram(&bd.hist);
|
||||
}
|
||||
|
||||
let height = start_height + i;
|
||||
let ref_bin = oracle.ref_bin();
|
||||
|
||||
if bd.high_bin <= 0.0 || bd.low_bin <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let err = if ref_bin < bd.high_bin {
|
||||
ref_bin - bd.high_bin
|
||||
} else if ref_bin > bd.low_bin {
|
||||
ref_bin - bd.low_bin
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
total_measured += 1;
|
||||
total_sq_err += err * err;
|
||||
let abs_err = err.abs();
|
||||
if abs_err > BINS_5PCT {
|
||||
gt_5 += 1;
|
||||
}
|
||||
if abs_err > BINS_10PCT {
|
||||
gt_10 += 1;
|
||||
}
|
||||
if abs_err > BINS_20PCT {
|
||||
gt_20 += 1;
|
||||
}
|
||||
if abs_err > worst_err {
|
||||
worst_err = abs_err;
|
||||
worst_height = height;
|
||||
}
|
||||
}
|
||||
|
||||
let rmse = if total_measured > 0 {
|
||||
bins_to_pct((total_sq_err / total_measured as f64).sqrt())
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
println!(
|
||||
"{:>8} {:>8} {:>7.3}% {:>7.1}% {:>6} {:>6} {:>6} {}",
|
||||
format!("{}k", start_height / 1000),
|
||||
total_measured,
|
||||
rmse,
|
||||
bins_to_pct(worst_err),
|
||||
gt_5,
|
||||
gt_10,
|
||||
gt_20,
|
||||
worst_height,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64());
|
||||
}
|
||||
@@ -6,8 +6,7 @@ use std::path::PathBuf;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{
|
||||
Config, Histogram, Oracle, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin,
|
||||
default_eligible_bin,
|
||||
bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW,
|
||||
};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
@@ -94,7 +93,11 @@ impl YearStats {
|
||||
fn median_pct(&mut self) -> f64 {
|
||||
self.errors.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let n = self.errors.len();
|
||||
if n == 0 { 0.0 } else { self.errors[n / 2] }
|
||||
if n == 0 {
|
||||
0.0
|
||||
} else {
|
||||
self.errors[n / 2]
|
||||
}
|
||||
}
|
||||
|
||||
fn percentile(&self, p: f64) -> f64 {
|
||||
@@ -172,15 +175,7 @@ fn main() {
|
||||
.map(|ts| (**ts / 86400).saturating_sub(GENESIS_DAY) as usize)
|
||||
.collect();
|
||||
|
||||
let start_price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(START_HEIGHT - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
|
||||
let config = Config::default();
|
||||
let mut oracle = Oracle::new(cents_to_bin(start_price * 100.0), config);
|
||||
let mut oracle = Oracle::from_seed();
|
||||
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
@@ -201,7 +196,11 @@ fn main() {
|
||||
let mut oracle_candles: Vec<DayCandle> = Vec::new();
|
||||
let mut current_di: Option<usize> = None;
|
||||
|
||||
for h in START_HEIGHT..total_heights {
|
||||
for h in START_HEIGHT_SLOW..total_heights {
|
||||
if h == START_HEIGHT_FAST {
|
||||
oracle.reconfigure(Config::default());
|
||||
}
|
||||
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
@@ -235,23 +234,18 @@ fn main() {
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
// Drop every output of a tx carrying an OP_RETURN (protocol machinery).
|
||||
let mut hist = Histogram::zeros();
|
||||
for tx in 0..tx_count {
|
||||
let tx_outputs = (0..tx_count).map(|tx| {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
if output_types[lo..hi].contains(&OutputType::OpReturn) {
|
||||
continue;
|
||||
}
|
||||
for i in lo..hi {
|
||||
if let Some(bin) = default_eligible_bin(values[i], output_types[i]) {
|
||||
hist.increment(bin as usize);
|
||||
}
|
||||
}
|
||||
}
|
||||
values[lo..hi]
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(output_types[lo..hi].iter().copied())
|
||||
});
|
||||
let hist = PaymentFilter::for_height(h).histogram(tx_outputs);
|
||||
|
||||
let ref_bin = oracle.process_histogram(&hist);
|
||||
let oracle_price = bin_to_cents(ref_bin) as f64 / 100.0;
|
||||
@@ -373,10 +367,12 @@ fn main() {
|
||||
println!(" brk_oracle accuracy report");
|
||||
println!(" ══════════════════════════");
|
||||
println!();
|
||||
println!(" Config: w12, alpha=2/7, search -9/+11, noisy/dust/round-btc filtered");
|
||||
println!(
|
||||
" Test range: height {} .. {} ({} blocks)",
|
||||
START_HEIGHT,
|
||||
" Config: slow w40 alpha=0.10 until {START_HEIGHT_FAST}, then w12 alpha=2/7; shared payment filter"
|
||||
);
|
||||
println!(
|
||||
" Test range: height {} .. {} ({} exchange-covered blocks)",
|
||||
START_HEIGHT_SLOW,
|
||||
total_heights - 1,
|
||||
overall.total_blocks
|
||||
);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,416 +0,0 @@
|
||||
//! Sweep round-value digit filter to find optimal configuration.
|
||||
//!
|
||||
//! Tests all 512 subsets of leading digits {1,...,9} to find which
|
||||
//! digits to filter out for best oracle accuracy.
|
||||
//!
|
||||
//! Phase 1: single pass over indexer, precompute per-block histograms.
|
||||
//! Phase 2: run 512 configs in parallel across CPU cores.
|
||||
//!
|
||||
//! Run with: cargo run -p brk_oracle --example sweep_digits --release
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, sats_to_bin};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
const BINS_5PCT: f64 = 4.24;
|
||||
const BINS_10PCT: f64 = 8.28;
|
||||
const BINS_20PCT: f64 = 15.84;
|
||||
|
||||
fn bins_to_pct(bins: f64) -> f64 {
|
||||
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
|
||||
}
|
||||
|
||||
fn seed_bin(start_height: usize) -> f64 {
|
||||
let price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(start_height - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
cents_to_bin(price * 100.0)
|
||||
}
|
||||
|
||||
fn leading_digit(sats: u64) -> u8 {
|
||||
let log = (sats as f64).log10();
|
||||
let magnitude = 10.0_f64.powf(log.floor());
|
||||
let d = (sats as f64 / magnitude).round() as u8;
|
||||
if d >= 10 { 1 } else { d }
|
||||
}
|
||||
|
||||
fn is_round(sats: u64) -> bool {
|
||||
let log = (sats as f64).log10();
|
||||
let magnitude = 10.0_f64.powf(log.floor());
|
||||
let leading = (sats as f64 / magnitude).round();
|
||||
let round_val = leading * magnitude;
|
||||
(sats as f64 - round_val).abs() <= round_val * 0.001
|
||||
}
|
||||
|
||||
fn mask_label(mask: u16) -> String {
|
||||
let digits: String = (1..=9u8)
|
||||
.filter(|&d| mask & (1 << (d - 1)) != 0)
|
||||
.map(|d| char::from_digit(d as u32, 10).unwrap())
|
||||
.collect();
|
||||
if digits.is_empty() {
|
||||
"none".to_string()
|
||||
} else {
|
||||
digits
|
||||
}
|
||||
}
|
||||
|
||||
struct Stats {
|
||||
total_sq_err: f64,
|
||||
total_bias: f64,
|
||||
max_err: f64,
|
||||
total_blocks: u64,
|
||||
gt_5pct: u64,
|
||||
gt_10pct: u64,
|
||||
gt_20pct: u64,
|
||||
}
|
||||
|
||||
impl Stats {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
total_sq_err: 0.0,
|
||||
total_bias: 0.0,
|
||||
max_err: 0.0,
|
||||
total_blocks: 0,
|
||||
gt_5pct: 0,
|
||||
gt_10pct: 0,
|
||||
gt_20pct: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, err: f64) {
|
||||
self.total_sq_err += err * err;
|
||||
self.total_bias += err;
|
||||
self.total_blocks += 1;
|
||||
let abs_err = err.abs();
|
||||
if abs_err > self.max_err {
|
||||
self.max_err = abs_err;
|
||||
}
|
||||
if abs_err > BINS_5PCT {
|
||||
self.gt_5pct += 1;
|
||||
}
|
||||
if abs_err > BINS_10PCT {
|
||||
self.gt_10pct += 1;
|
||||
}
|
||||
if abs_err > BINS_20PCT {
|
||||
self.gt_20pct += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn rmse_pct(&self) -> f64 {
|
||||
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
|
||||
}
|
||||
|
||||
fn max_pct(&self) -> f64 {
|
||||
bins_to_pct(self.max_err)
|
||||
}
|
||||
|
||||
fn bias(&self) -> f64 {
|
||||
self.total_bias / self.total_blocks as f64
|
||||
}
|
||||
}
|
||||
|
||||
struct BlockData {
|
||||
full_hist: Histogram,
|
||||
/// (bin_index, leading_digit) for outputs that are round values.
|
||||
round_outputs: Vec<(u16, u8)>,
|
||||
high_bin: f64,
|
||||
low_bin: f64,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let t0 = Instant::now();
|
||||
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| {
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
PathBuf::from(home).join(".brk")
|
||||
});
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("Failed to read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("Failed to parse height OHLC");
|
||||
|
||||
let height_bands: Vec<(f64, f64)> = height_ohlc
|
||||
.iter()
|
||||
.map(|ohlc| {
|
||||
let high = ohlc[1];
|
||||
let low = ohlc[2];
|
||||
if high > 0.0 && low > 0.0 {
|
||||
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let sweep_start: usize = 575_000;
|
||||
|
||||
// Phase 1: precompute per-block data in a single pass over the indexer.
|
||||
eprintln!("Phase 1: precomputing block data...");
|
||||
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
|
||||
let ref_config = Config::default();
|
||||
let total_blocks = total_heights - sweep_start;
|
||||
let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
|
||||
|
||||
for h in sweep_start..total_heights {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
|
||||
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
|
||||
indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.collect_one(ft + 1)
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
} else {
|
||||
out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize()
|
||||
};
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
let mut full_hist = Histogram::zeros();
|
||||
let mut round_outputs = Vec::new();
|
||||
|
||||
for (sats, output_type) in values.into_iter().zip(output_types) {
|
||||
if ref_config.excluded_output_types.contains(&output_type) {
|
||||
continue;
|
||||
}
|
||||
if *sats < ref_config.min_sats {
|
||||
continue;
|
||||
}
|
||||
if let Some(bin) = sats_to_bin(sats) {
|
||||
full_hist.increment(bin);
|
||||
if is_round(*sats) {
|
||||
let d = leading_digit(*sats);
|
||||
if (1..=9).contains(&d) {
|
||||
round_outputs.push((bin as u16, d));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (high_bin, low_bin) = if h < height_bands.len() {
|
||||
height_bands[h]
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
};
|
||||
|
||||
blocks.push(BlockData {
|
||||
full_hist,
|
||||
round_outputs,
|
||||
high_bin,
|
||||
low_bin,
|
||||
});
|
||||
|
||||
if (h - sweep_start).is_multiple_of(50_000) {
|
||||
eprint!(
|
||||
"\r {}/{} ({:.0}%)",
|
||||
h - sweep_start,
|
||||
total_blocks,
|
||||
(h - sweep_start) as f64 / total_blocks as f64 * 100.0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mem_hists = blocks.len() * std::mem::size_of::<Histogram>();
|
||||
let mem_rounds: usize = blocks.iter().map(|b| b.round_outputs.len() * 3).sum();
|
||||
eprintln!(
|
||||
"\r {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s",
|
||||
blocks.len(),
|
||||
mem_hists as f64 / 1e9,
|
||||
mem_rounds as f64 / 1e6,
|
||||
t0.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
// Phase 2: sweep digit masks in parallel.
|
||||
// Always filter digit 1 (powers of 10), sweep digits 2-9.
|
||||
let base_mask: u16 = 1 << 0; // digit 1 always on
|
||||
let num_masks: usize = 256; // 2^8 subsets of {2,...,9}
|
||||
let num_threads = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(8);
|
||||
eprintln!(
|
||||
"Phase 2: sweeping {} masks across {} threads...",
|
||||
num_masks, num_threads
|
||||
);
|
||||
|
||||
let t1 = Instant::now();
|
||||
let blocks = &blocks; // shared reference for threads
|
||||
|
||||
let all_results: Vec<(u16, Stats)> = std::thread::scope(|s| {
|
||||
let masks_per_thread = num_masks.div_ceil(num_threads);
|
||||
|
||||
let handles: Vec<_> = (0..num_threads)
|
||||
.map(|t| {
|
||||
s.spawn(move || {
|
||||
let mask_start = t * masks_per_thread;
|
||||
let mask_end = ((t + 1) * masks_per_thread).min(num_masks);
|
||||
let mut results = Vec::with_capacity(mask_end - mask_start);
|
||||
|
||||
for idx in mask_start..mask_end {
|
||||
// Shift idx bits into positions 1-8 (digits 2-9) and add base_mask (digit 1).
|
||||
let mask = base_mask | ((idx as u16) << 1);
|
||||
let mut oracle = Oracle::new(
|
||||
seed_bin(sweep_start),
|
||||
Config {
|
||||
exclude_common_round_values: false,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let mut stats = Stats::new();
|
||||
|
||||
for bd in blocks.iter() {
|
||||
let mut hist = bd.full_hist.clone();
|
||||
for &(bin, digit) in &bd.round_outputs {
|
||||
if mask & (1 << (digit - 1)) != 0 {
|
||||
hist[bin as usize] -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
let ref_bin = oracle.process_histogram(&hist);
|
||||
|
||||
if bd.high_bin > 0.0 && bd.low_bin > 0.0 {
|
||||
let err = if ref_bin < bd.high_bin {
|
||||
ref_bin - bd.high_bin
|
||||
} else if ref_bin > bd.low_bin {
|
||||
ref_bin - bd.low_bin
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
stats.update(err);
|
||||
}
|
||||
}
|
||||
|
||||
results.push((mask, stats));
|
||||
}
|
||||
|
||||
results
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
handles
|
||||
.into_iter()
|
||||
.flat_map(|h| h.join().unwrap())
|
||||
.collect()
|
||||
});
|
||||
|
||||
eprintln!(" Done in {:.1}s.", t1.elapsed().as_secs_f64());
|
||||
|
||||
// Sort by RMSE.
|
||||
let mut results: Vec<&(u16, Stats)> = all_results.iter().collect();
|
||||
results.sort_by(|a, b| a.1.rmse_pct().partial_cmp(&b.1.rmse_pct()).unwrap());
|
||||
|
||||
// Print top 20.
|
||||
println!();
|
||||
println!("Top 20 (by RMSE):");
|
||||
println!(
|
||||
"{:>4} {:>12} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
|
||||
"#", "Digits", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
|
||||
);
|
||||
println!("{}", "-".repeat(70));
|
||||
for (rank, (mask, s)) in results.iter().take(20).enumerate() {
|
||||
println!(
|
||||
"{:>4} {:>12} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}",
|
||||
rank + 1,
|
||||
mask_label(*mask),
|
||||
s.rmse_pct(),
|
||||
s.max_pct(),
|
||||
s.gt_5pct,
|
||||
s.gt_10pct,
|
||||
s.gt_20pct,
|
||||
s.bias()
|
||||
);
|
||||
}
|
||||
|
||||
// Print bottom 5.
|
||||
println!();
|
||||
println!("Bottom 5 (worst):");
|
||||
println!(
|
||||
"{:>4} {:>12} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
|
||||
"#", "Digits", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
|
||||
);
|
||||
println!("{}", "-".repeat(70));
|
||||
for (mask, s) in results.iter().rev().take(5) {
|
||||
println!(
|
||||
"{:>4} {:>12} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}",
|
||||
"",
|
||||
mask_label(*mask),
|
||||
s.rmse_pct(),
|
||||
s.max_pct(),
|
||||
s.gt_5pct,
|
||||
s.gt_10pct,
|
||||
s.gt_20pct,
|
||||
s.bias()
|
||||
);
|
||||
}
|
||||
|
||||
// Print current config {1,2,3,5} for reference.
|
||||
let current_mask: u16 = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 4); // digits 1,2,3,5
|
||||
let current_stats = all_results
|
||||
.iter()
|
||||
.find(|(m, _)| *m == current_mask)
|
||||
.map(|(_, s)| s)
|
||||
.unwrap();
|
||||
let current_rank = results
|
||||
.iter()
|
||||
.position(|(m, _)| *m == current_mask)
|
||||
.unwrap();
|
||||
println!();
|
||||
println!(
|
||||
"Current {{1,2,3,5}} = rank {}/{}: RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
|
||||
current_rank + 1,
|
||||
num_masks,
|
||||
current_stats.rmse_pct(),
|
||||
current_stats.max_pct(),
|
||||
current_stats.gt_5pct,
|
||||
current_stats.gt_10pct,
|
||||
current_stats.gt_20pct,
|
||||
);
|
||||
|
||||
println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64());
|
||||
}
|
||||
@@ -1,452 +0,0 @@
|
||||
//! Sweep round-value tolerance to find optimal rounding threshold.
|
||||
//!
|
||||
//! Tests different tolerance percentages (0%, 0.01%, 0.1%, 1%, etc.) for
|
||||
//! detecting round BTC amounts, combined with several digit filter masks.
|
||||
//!
|
||||
//! Phase 1: single pass over indexer, store per-output relative errors.
|
||||
//! Phase 2: sweep tolerance × mask combos across CPU cores.
|
||||
//!
|
||||
//! Run with: cargo run -p brk_oracle --example sweep_tolerance --release
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, sats_to_bin};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
const BINS_5PCT: f64 = 4.24;
|
||||
const BINS_10PCT: f64 = 8.28;
|
||||
const BINS_20PCT: f64 = 15.84;
|
||||
|
||||
fn bins_to_pct(bins: f64) -> f64 {
|
||||
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
|
||||
}
|
||||
|
||||
fn seed_bin(start_height: usize) -> f64 {
|
||||
let price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(start_height - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
cents_to_bin(price * 100.0)
|
||||
}
|
||||
|
||||
fn leading_digit(sats: u64) -> u8 {
|
||||
let log = (sats as f64).log10();
|
||||
let magnitude = 10.0_f64.powf(log.floor());
|
||||
let d = (sats as f64 / magnitude).round() as u8;
|
||||
if d >= 10 { 1 } else { d }
|
||||
}
|
||||
|
||||
/// Returns the relative error of `sats` from its nearest round value (d × 10^n).
|
||||
/// e.g. 10_050 → leading=1, round_val=10_000, rel_err = 50/10000 = 0.005
|
||||
fn relative_roundness(sats: u64) -> f64 {
|
||||
let log = (sats as f64).log10();
|
||||
let magnitude = 10.0_f64.powf(log.floor());
|
||||
let leading = (sats as f64 / magnitude).round();
|
||||
let round_val = leading * magnitude;
|
||||
(sats as f64 - round_val).abs() / round_val
|
||||
}
|
||||
|
||||
struct Stats {
|
||||
total_sq_err: f64,
|
||||
total_bias: f64,
|
||||
max_err: f64,
|
||||
total_blocks: u64,
|
||||
gt_5pct: u64,
|
||||
gt_10pct: u64,
|
||||
gt_20pct: u64,
|
||||
}
|
||||
|
||||
impl Stats {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
total_sq_err: 0.0,
|
||||
total_bias: 0.0,
|
||||
max_err: 0.0,
|
||||
total_blocks: 0,
|
||||
gt_5pct: 0,
|
||||
gt_10pct: 0,
|
||||
gt_20pct: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, err: f64) {
|
||||
self.total_sq_err += err * err;
|
||||
self.total_bias += err;
|
||||
self.total_blocks += 1;
|
||||
let abs_err = err.abs();
|
||||
if abs_err > self.max_err {
|
||||
self.max_err = abs_err;
|
||||
}
|
||||
if abs_err > BINS_5PCT {
|
||||
self.gt_5pct += 1;
|
||||
}
|
||||
if abs_err > BINS_10PCT {
|
||||
self.gt_10pct += 1;
|
||||
}
|
||||
if abs_err > BINS_20PCT {
|
||||
self.gt_20pct += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn rmse_pct(&self) -> f64 {
|
||||
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
|
||||
}
|
||||
|
||||
fn max_pct(&self) -> f64 {
|
||||
bins_to_pct(self.max_err)
|
||||
}
|
||||
|
||||
fn bias(&self) -> f64 {
|
||||
self.total_bias / self.total_blocks as f64
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-output data: bin index, leading digit, relative error from round value.
|
||||
struct RoundOutput {
|
||||
bin: u16,
|
||||
digit: u8,
|
||||
rel_err: f32, // f32 is plenty of precision, saves memory
|
||||
}
|
||||
|
||||
struct BlockData {
|
||||
full_hist: Histogram,
|
||||
round_outputs: Vec<RoundOutput>,
|
||||
high_bin: f64,
|
||||
low_bin: f64,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let t0 = Instant::now();
|
||||
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| {
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
PathBuf::from(home).join(".brk")
|
||||
});
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("Failed to read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("Failed to parse height OHLC");
|
||||
|
||||
let height_bands: Vec<(f64, f64)> = height_ohlc
|
||||
.iter()
|
||||
.map(|ohlc| {
|
||||
let high = ohlc[1];
|
||||
let low = ohlc[2];
|
||||
if high > 0.0 && low > 0.0 {
|
||||
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let sweep_start: usize = 575_000;
|
||||
|
||||
// Phase 1: precompute per-block data.
|
||||
// Store all potentially-round outputs with their relative error so we can
|
||||
// filter at different tolerance thresholds in Phase 2.
|
||||
eprintln!("Phase 1: precomputing block data...");
|
||||
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
|
||||
let ref_config = Config::default();
|
||||
let total_blocks = total_heights - sweep_start;
|
||||
let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
|
||||
|
||||
// Use the widest tolerance we'll test (5%) to decide what to store.
|
||||
// Outputs beyond 5% relative error will never be filtered at any tolerance.
|
||||
let max_tolerance: f64 = 0.05;
|
||||
|
||||
for h in sweep_start..total_heights {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
|
||||
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
|
||||
indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.collect_one(ft + 1)
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
} else {
|
||||
out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize()
|
||||
};
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
let mut full_hist = Histogram::zeros();
|
||||
let mut round_outputs = Vec::new();
|
||||
|
||||
for (sats, output_type) in values.into_iter().zip(output_types) {
|
||||
if ref_config.excluded_output_types.contains(&output_type) {
|
||||
continue;
|
||||
}
|
||||
if *sats < ref_config.min_sats {
|
||||
continue;
|
||||
}
|
||||
if let Some(bin) = sats_to_bin(sats) {
|
||||
full_hist.increment(bin);
|
||||
let d = leading_digit(*sats);
|
||||
if (1..=9).contains(&d) {
|
||||
let rel_err = relative_roundness(*sats);
|
||||
if rel_err <= max_tolerance {
|
||||
round_outputs.push(RoundOutput {
|
||||
bin: bin as u16,
|
||||
digit: d,
|
||||
rel_err: rel_err as f32,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (high_bin, low_bin) = if h < height_bands.len() {
|
||||
height_bands[h]
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
};
|
||||
|
||||
blocks.push(BlockData {
|
||||
full_hist,
|
||||
round_outputs,
|
||||
high_bin,
|
||||
low_bin,
|
||||
});
|
||||
|
||||
if (h - sweep_start).is_multiple_of(50_000) {
|
||||
eprint!(
|
||||
"\r {}/{} ({:.0}%)",
|
||||
h - sweep_start,
|
||||
total_blocks,
|
||||
(h - sweep_start) as f64 / total_blocks as f64 * 100.0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mem_hists = blocks.len() * std::mem::size_of::<Histogram>();
|
||||
let mem_rounds: usize = blocks
|
||||
.iter()
|
||||
.map(|b| b.round_outputs.len() * std::mem::size_of::<RoundOutput>())
|
||||
.sum();
|
||||
eprintln!(
|
||||
"\r {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s",
|
||||
blocks.len(),
|
||||
mem_hists as f64 / 1e9,
|
||||
mem_rounds as f64 / 1e6,
|
||||
t0.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
// Phase 2: sweep tolerance × mask combos.
|
||||
// Tolerances as fractions (not percentages).
|
||||
let tolerances: &[(f64, &str)] = &[
|
||||
(0.0, "0%"),
|
||||
(0.0001, "0.01%"),
|
||||
(0.0005, "0.05%"),
|
||||
(0.001, "0.1%"),
|
||||
(0.002, "0.2%"),
|
||||
(0.005, "0.5%"),
|
||||
(0.01, "1%"),
|
||||
(0.02, "2%"),
|
||||
(0.05, "5%"),
|
||||
];
|
||||
|
||||
// 987654321
|
||||
let masks: &[(u16, &str)] = &[
|
||||
(0b0_0000_0000, "none"),
|
||||
(0b0_0001_0111, "{1,2,3,5}"),
|
||||
(0b0_0001_1111, "{1,2,3,4,5}"),
|
||||
(0b0_0011_0111, "{1,2,3,5,6}"),
|
||||
(0b0_0111_0111, "{1,2,3,5,6,7}"),
|
||||
(0b1_1111_1111, "{1-9}"),
|
||||
];
|
||||
|
||||
let num_configs = tolerances.len() * masks.len();
|
||||
let num_threads = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(8);
|
||||
eprintln!(
|
||||
"Phase 2: sweeping {} configs ({} tolerances × {} masks) across {} threads...",
|
||||
num_configs,
|
||||
tolerances.len(),
|
||||
masks.len(),
|
||||
num_threads
|
||||
);
|
||||
|
||||
let t1 = Instant::now();
|
||||
let blocks = &blocks;
|
||||
let tolerances_ref = tolerances;
|
||||
let masks_ref = masks;
|
||||
|
||||
let all_results: Vec<(usize, usize, Stats)> = std::thread::scope(|s| {
|
||||
let configs_per_thread = num_configs.div_ceil(num_threads);
|
||||
|
||||
let handles: Vec<_> = (0..num_threads)
|
||||
.map(|t| {
|
||||
s.spawn(move || {
|
||||
let cfg_start = t * configs_per_thread;
|
||||
let cfg_end = ((t + 1) * configs_per_thread).min(num_configs);
|
||||
if cfg_start >= cfg_end {
|
||||
return vec![];
|
||||
}
|
||||
let mut results = Vec::with_capacity(cfg_end - cfg_start);
|
||||
|
||||
for cfg_idx in cfg_start..cfg_end {
|
||||
let ti = cfg_idx / masks_ref.len();
|
||||
let mi = cfg_idx % masks_ref.len();
|
||||
let (tolerance, _) = tolerances_ref[ti];
|
||||
let (mask, _) = masks_ref[mi];
|
||||
|
||||
let mut oracle = Oracle::new(
|
||||
seed_bin(sweep_start),
|
||||
Config {
|
||||
exclude_common_round_values: false,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let mut stats = Stats::new();
|
||||
|
||||
for bd in blocks.iter() {
|
||||
let mut hist = bd.full_hist.clone();
|
||||
|
||||
// Remove outputs matching this tolerance + mask.
|
||||
let tol_f32 = tolerance as f32;
|
||||
for ro in &bd.round_outputs {
|
||||
if mask & (1 << (ro.digit - 1)) != 0 && ro.rel_err <= tol_f32 {
|
||||
hist[ro.bin as usize] -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
let ref_bin = oracle.process_histogram(&hist);
|
||||
|
||||
if bd.high_bin > 0.0 && bd.low_bin > 0.0 {
|
||||
let err = if ref_bin < bd.high_bin {
|
||||
ref_bin - bd.high_bin
|
||||
} else if ref_bin > bd.low_bin {
|
||||
ref_bin - bd.low_bin
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
stats.update(err);
|
||||
}
|
||||
}
|
||||
|
||||
results.push((ti, mi, stats));
|
||||
}
|
||||
|
||||
results
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
handles
|
||||
.into_iter()
|
||||
.flat_map(|h| h.join().unwrap())
|
||||
.collect()
|
||||
});
|
||||
|
||||
eprintln!(" Done in {:.1}s.", t1.elapsed().as_secs_f64());
|
||||
|
||||
// Print results grouped by tolerance.
|
||||
println!();
|
||||
println!(
|
||||
"{:>8} {:>16} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
|
||||
"Tol", "Digits", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
|
||||
);
|
||||
println!("{}", "-".repeat(88));
|
||||
|
||||
for (ti, &(_, tol_label)) in tolerances.iter().enumerate() {
|
||||
for (mi, &(_, mask_label)) in masks.iter().enumerate() {
|
||||
let (_, _, stats) = all_results
|
||||
.iter()
|
||||
.find(|(t, m, _)| *t == ti && *m == mi)
|
||||
.unwrap();
|
||||
println!(
|
||||
"{:>8} {:>16} {:>8} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}",
|
||||
tol_label,
|
||||
mask_label,
|
||||
stats.total_blocks,
|
||||
stats.rmse_pct(),
|
||||
stats.max_pct(),
|
||||
stats.gt_5pct,
|
||||
stats.gt_10pct,
|
||||
stats.gt_20pct,
|
||||
stats.bias()
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
// Find overall best config by RMSE.
|
||||
let best = all_results
|
||||
.iter()
|
||||
.min_by(|a, b| a.2.rmse_pct().partial_cmp(&b.2.rmse_pct()).unwrap())
|
||||
.unwrap();
|
||||
let (bti, bmi, bs) = best;
|
||||
println!(
|
||||
"Best: tolerance={}, digits={} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
|
||||
tolerances[*bti].1,
|
||||
masks[*bmi].1,
|
||||
bs.rmse_pct(),
|
||||
bs.max_pct(),
|
||||
bs.gt_5pct,
|
||||
bs.gt_10pct,
|
||||
bs.gt_20pct,
|
||||
);
|
||||
|
||||
// Show current config for reference.
|
||||
let current = all_results
|
||||
.iter()
|
||||
.find(|(t, m, _)| tolerances[*t].0 == 0.001 && masks[*m].0 == 0b0_0011_0111)
|
||||
.unwrap();
|
||||
let (_, _, cs) = current;
|
||||
println!(
|
||||
"Current: tolerance=0.1%, digits={{1,2,3,5,6}} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
|
||||
cs.rmse_pct(),
|
||||
cs.max_pct(),
|
||||
cs.gt_5pct,
|
||||
cs.gt_10pct,
|
||||
cs.gt_20pct,
|
||||
);
|
||||
|
||||
println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64());
|
||||
}
|
||||
@@ -1,286 +0,0 @@
|
||||
//! Validate oracle accuracy against exchange reference prices.
|
||||
//!
|
||||
//! Run with: cargo run -p brk_oracle --example validate --release
|
||||
//!
|
||||
//! Requires:
|
||||
//! - ~/.brk indexed blockchain data (brk_indexer)
|
||||
//! - examples/height_price_ohlc.json (per-height [open, high, low, close] in dollars)
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{
|
||||
Config, Histogram, Oracle, PRICES, START_HEIGHT, cents_to_bin, default_eligible_bin,
|
||||
};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
const BINS_5PCT: f64 = 4.24;
|
||||
const BINS_10PCT: f64 = 8.28;
|
||||
const BINS_20PCT: f64 = 15.84;
|
||||
|
||||
fn bins_to_pct(bins: f64) -> f64 {
|
||||
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
|
||||
}
|
||||
|
||||
fn seed_bin(start_height: usize) -> f64 {
|
||||
let price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(start_height - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
cents_to_bin(price * 100.0)
|
||||
}
|
||||
|
||||
struct Stats {
|
||||
total_sq_err: f64,
|
||||
total_bias: f64,
|
||||
max_err: f64,
|
||||
total_blocks: u64,
|
||||
gt_5pct: u64,
|
||||
gt_10pct: u64,
|
||||
gt_20pct: u64,
|
||||
}
|
||||
|
||||
impl Stats {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
total_sq_err: 0.0,
|
||||
total_bias: 0.0,
|
||||
max_err: 0.0,
|
||||
total_blocks: 0,
|
||||
gt_5pct: 0,
|
||||
gt_10pct: 0,
|
||||
gt_20pct: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, err: f64) {
|
||||
self.total_sq_err += err * err;
|
||||
self.total_bias += err;
|
||||
self.total_blocks += 1;
|
||||
let abs_err = err.abs();
|
||||
if abs_err > self.max_err {
|
||||
self.max_err = abs_err;
|
||||
}
|
||||
if abs_err > BINS_5PCT {
|
||||
self.gt_5pct += 1;
|
||||
}
|
||||
if abs_err > BINS_10PCT {
|
||||
self.gt_10pct += 1;
|
||||
}
|
||||
if abs_err > BINS_20PCT {
|
||||
self.gt_20pct += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn rmse_pct(&self) -> f64 {
|
||||
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
|
||||
}
|
||||
|
||||
fn max_pct(&self) -> f64 {
|
||||
bins_to_pct(self.max_err)
|
||||
}
|
||||
|
||||
fn bias(&self) -> f64 {
|
||||
self.total_bias / self.total_blocks as f64
|
||||
}
|
||||
}
|
||||
|
||||
struct Run {
|
||||
label: &'static str,
|
||||
start_height: usize,
|
||||
oracle: Option<Oracle>,
|
||||
stats: Stats,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| {
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
PathBuf::from(home).join(".brk")
|
||||
});
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("Failed to read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("Failed to parse height OHLC");
|
||||
|
||||
// Pre-compute per-height (high_bin, low_bin) tolerance band.
|
||||
let height_bands: Vec<(f64, f64)> = height_ohlc
|
||||
.iter()
|
||||
.map(|ohlc| {
|
||||
let high = ohlc[1];
|
||||
let low = ohlc[2];
|
||||
if high > 0.0 && low > 0.0 {
|
||||
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut runs = vec![
|
||||
Run {
|
||||
label: "w12 @ 575k",
|
||||
start_height: 575_000,
|
||||
oracle: None,
|
||||
stats: Stats::new(),
|
||||
},
|
||||
Run {
|
||||
label: "w12 @ 600k",
|
||||
start_height: 600_000,
|
||||
oracle: None,
|
||||
stats: Stats::new(),
|
||||
},
|
||||
Run {
|
||||
label: "w12 @ 630k",
|
||||
start_height: 630_000,
|
||||
oracle: None,
|
||||
stats: Stats::new(),
|
||||
},
|
||||
];
|
||||
|
||||
// Build per-block filtered histograms from the indexer, feeding all oracles in one pass.
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
|
||||
// Pre-collect height-indexed vecs (small). Transaction-indexed vecs are too large.
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
|
||||
for h in START_HEIGHT..total_heights {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
|
||||
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
|
||||
indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.collect_one(ft + 1)
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
} else {
|
||||
out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize()
|
||||
};
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
// Build filtered histogram once for all oracles.
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
let mut hist = Histogram::zeros();
|
||||
for (sats, output_type) in values.into_iter().zip(output_types) {
|
||||
if let Some(bin) = default_eligible_bin(sats, output_type) {
|
||||
hist.increment(bin as usize);
|
||||
}
|
||||
}
|
||||
|
||||
for run in &mut runs {
|
||||
if h < run.start_height {
|
||||
continue;
|
||||
}
|
||||
if run.oracle.is_none() {
|
||||
let config = Config::default();
|
||||
run.oracle = Some(Oracle::new(seed_bin(run.start_height), config));
|
||||
}
|
||||
let ref_bin = run.oracle.as_mut().unwrap().process_histogram(&hist);
|
||||
|
||||
if h < height_bands.len() {
|
||||
let (high_bin, low_bin) = height_bands[h];
|
||||
if high_bin > 0.0 && low_bin > 0.0 {
|
||||
let err = if ref_bin < high_bin {
|
||||
ref_bin - high_bin
|
||||
} else if ref_bin > low_bin {
|
||||
ref_bin - low_bin
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
run.stats.update(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Print results.
|
||||
println!();
|
||||
println!(
|
||||
"{:<14} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
|
||||
"Config", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
|
||||
);
|
||||
println!("{}", "-".repeat(72));
|
||||
for run in &runs {
|
||||
let s = &run.stats;
|
||||
println!(
|
||||
"{:<14} {:>8} {:>7.2}% {:>7.1}% {:>6} {:>6} {:>6} {:>+8.2}",
|
||||
run.label,
|
||||
s.total_blocks,
|
||||
s.rmse_pct(),
|
||||
s.max_pct(),
|
||||
s.gt_5pct,
|
||||
s.gt_10pct,
|
||||
s.gt_20pct,
|
||||
s.bias()
|
||||
);
|
||||
}
|
||||
println!();
|
||||
|
||||
// Verify exact counts against reference.
|
||||
// Reference: trunc w12 @ 575k: 261 >5%, 40 >10%, 0 >20%
|
||||
// trunc w12 @ 600k: 174 >5%, 31 >10%, 0 >20%
|
||||
// trunc w12 @ 630k: 84 >5%, 9 >10%, 0 >20%
|
||||
let expected: &[(&str, u64, u64, u64)] = &[
|
||||
("w12 @ 575k", 237, 22, 0),
|
||||
("w12 @ 600k", 152, 15, 0),
|
||||
("w12 @ 630k", 84, 9, 0),
|
||||
];
|
||||
|
||||
for (run, &(label, exp_5, exp_10, exp_20)) in runs.iter().zip(expected) {
|
||||
let s = &run.stats;
|
||||
assert_eq!(
|
||||
s.gt_20pct, exp_20,
|
||||
"{label}: expected {exp_20} blocks >20%, got {}",
|
||||
s.gt_20pct
|
||||
);
|
||||
assert_eq!(
|
||||
s.gt_10pct, exp_10,
|
||||
"{label}: expected {exp_10} blocks >10%, got {}",
|
||||
s.gt_10pct
|
||||
);
|
||||
assert_eq!(
|
||||
s.gt_5pct, exp_5,
|
||||
"{label}: expected {exp_5} blocks >5%, got {}",
|
||||
s.gt_5pct
|
||||
);
|
||||
}
|
||||
|
||||
println!("All assertions passed!");
|
||||
}
|
||||
@@ -1,13 +1,17 @@
|
||||
use brk_types::OutputType;
|
||||
use std::ops::Range;
|
||||
|
||||
/// Dust floor used by `Config::default()` and `default_eligible_bin`.
|
||||
pub(crate) const DEFAULT_MIN_SATS: u64 = 1000;
|
||||
/// First height the oracle computes on-chain, with the slow cold-start EMA
|
||||
/// ([`slow`](Config::slow)). Below it, prices come from
|
||||
/// [`pre_oracle_prices_from`](crate::pre_oracle_prices_from).
|
||||
pub const START_HEIGHT_SLOW: usize = 340_000;
|
||||
|
||||
/// Output types skipped by `Config::default()` (protocol-dominated) and the
|
||||
/// source of truth for `default_eligible_bin`'s precomputed exclusion mask.
|
||||
pub(crate) const DEFAULT_EXCLUDED_OUTPUT_TYPES: &[OutputType] = &[OutputType::P2TR];
|
||||
/// Height where the oracle switches slow -> fast EMA ([`default`](Config::default)).
|
||||
/// The regimes are complementary: slow resists the round-USD half-price drift
|
||||
/// that locks fast below here, while fast tracks the 2018-2019 crashes that lock
|
||||
/// slow.
|
||||
pub const START_HEIGHT_FAST: usize = 508_000;
|
||||
|
||||
#[derive(Clone)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Config {
|
||||
/// EMA decay: 2/(N+1) where N is span in blocks. 2/7 = 6-block span.
|
||||
pub alpha: f64,
|
||||
@@ -16,12 +20,12 @@ pub struct Config {
|
||||
/// Search window bins below/above previous estimate. Asymmetric for log-scale.
|
||||
pub search_below: usize,
|
||||
pub search_above: usize,
|
||||
/// Minimum output value in sats (dust filter).
|
||||
pub min_sats: u64,
|
||||
/// Exclude round BTC amounts that create false stencil matches.
|
||||
pub exclude_common_round_values: bool,
|
||||
/// Output types to ignore (e.g. P2TR, P2WSH are noisy).
|
||||
pub excluded_output_types: Vec<OutputType>,
|
||||
/// Weight of the adaptive shape-anchoring restoring force added to the
|
||||
/// stencil score. `0.0` disables it (mature regime, where the fast EMA
|
||||
/// tracks real moves the shape term would resist). The slow cold-start uses
|
||||
/// a positive weight to resist round-USD octave aliasing in the thin early
|
||||
/// output mix.
|
||||
pub shape_weight: f64,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
@@ -29,11 +33,78 @@ impl Default for Config {
|
||||
Self {
|
||||
alpha: 2.0 / 7.0,
|
||||
window_size: 12,
|
||||
search_below: 9,
|
||||
search_below: 12,
|
||||
search_above: 11,
|
||||
min_sats: DEFAULT_MIN_SATS,
|
||||
exclude_common_round_values: true,
|
||||
excluded_output_types: DEFAULT_EXCLUDED_OUTPUT_TYPES.to_vec(),
|
||||
shape_weight: 0.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Cold-start config below [`START_HEIGHT_FAST`]: a slow EMA
|
||||
/// (span ~19) that resists the round-USD half-price drift the fast default
|
||||
/// octave-locks onto in the thin pre-2018 output mix. Window grows to 40 to
|
||||
/// hold the decay, and a shape-anchoring restoring force (`shape_weight`)
|
||||
/// pulls the pick toward the octave whose arm-shape looks like real payments.
|
||||
pub fn slow() -> Self {
|
||||
Self {
|
||||
alpha: 0.10,
|
||||
window_size: 40,
|
||||
shape_weight: 8.0,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Config for `height`: [`slow`](Self::slow) below [`START_HEIGHT_FAST`], else
|
||||
/// [`default`](Self::default).
|
||||
pub fn for_height(height: usize) -> Self {
|
||||
if height < START_HEIGHT_FAST {
|
||||
Self::slow()
|
||||
} else {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a block range into sub-ranges with a single EMA configuration.
|
||||
pub fn segments_for_range(range: Range<usize>) -> impl Iterator<Item = Range<usize>> {
|
||||
let split = START_HEIGHT_FAST.max(range.start).min(range.end);
|
||||
[range.start..split, split..range.end]
|
||||
.into_iter()
|
||||
.filter(|range| !range.is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn segments_for_range_splits_at_fast_start() {
|
||||
let segments: Vec<_> =
|
||||
Config::segments_for_range((START_HEIGHT_FAST - 2)..(START_HEIGHT_FAST + 2)).collect();
|
||||
assert_eq!(
|
||||
segments,
|
||||
vec![
|
||||
(START_HEIGHT_FAST - 2)..START_HEIGHT_FAST,
|
||||
START_HEIGHT_FAST..(START_HEIGHT_FAST + 2),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segments_for_range_omits_empty_sides() {
|
||||
let slow: Vec<_> =
|
||||
Config::segments_for_range((START_HEIGHT_FAST - 2)..START_HEIGHT_FAST).collect();
|
||||
assert_eq!(slow, vec![(START_HEIGHT_FAST - 2)..START_HEIGHT_FAST]);
|
||||
|
||||
let fast: Vec<_> =
|
||||
Config::segments_for_range(START_HEIGHT_FAST..(START_HEIGHT_FAST + 2)).collect();
|
||||
assert_eq!(fast, vec![START_HEIGHT_FAST..(START_HEIGHT_FAST + 2)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn for_height_selects_regime() {
|
||||
assert_eq!(Config::for_height(START_HEIGHT_FAST - 1), Config::slow());
|
||||
assert_eq!(Config::for_height(START_HEIGHT_FAST), Config::default());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
use brk_types::{OutputType, Sats};
|
||||
|
||||
use crate::scale::{sats_to_bin, HistogramRaw};
|
||||
|
||||
/// Dust floor: outputs below this many sats are too small to be payments.
|
||||
const MIN_SATS: u64 = 1000;
|
||||
|
||||
/// Output types skipped entirely (protocol-dominated, too noisy to carry the
|
||||
/// round-dollar signal).
|
||||
const EXCLUDED_OUTPUT_TYPES: &[OutputType] = &[OutputType::P2TR];
|
||||
|
||||
/// Bitmask form of [`EXCLUDED_OUTPUT_TYPES`], folded at compile time so
|
||||
/// [`PaymentFilter::eligible_bin`] checks membership with a single AND.
|
||||
const EXCLUDED_MASK: u16 = {
|
||||
let mut mask = 0u16;
|
||||
let mut i = 0;
|
||||
while i < EXCLUDED_OUTPUT_TYPES.len() {
|
||||
mask |= 1u16 << EXCLUDED_OUTPUT_TYPES[i] as u8;
|
||||
i += 1;
|
||||
}
|
||||
mask
|
||||
};
|
||||
|
||||
/// Round-dollar payment filter.
|
||||
///
|
||||
/// Input: transaction outputs. Output: eligible log-scale bins or a fresh block
|
||||
/// histogram. The only state is the transaction-output fan-out cap selected by
|
||||
/// block height, or [`MODERN`](Self::MODERN) for live modern transaction streams.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct PaymentFilter {
|
||||
tx_output_fanout_cap: usize,
|
||||
}
|
||||
|
||||
impl PaymentFilter {
|
||||
/// Pre-modern transaction-output fan-out cap. Above this, the transaction is
|
||||
/// a batch payout (exchange sweep, mixer fan-out), not a round-dollar
|
||||
/// payment.
|
||||
pub const PRE_MODERN_TX_OUTPUT_FANOUT_CAP: usize = 100;
|
||||
|
||||
/// Modern-chain transaction-output fan-out cap. Dense post-630k blocks can
|
||||
/// carry more genuine payment outputs, but very large fan-outs can still
|
||||
/// dominate one EMA slot and create a false round-dollar ladder.
|
||||
pub const MODERN_TX_OUTPUT_FANOUT_CAP: usize = 250;
|
||||
|
||||
/// Height where [`Self::PRE_MODERN_TX_OUTPUT_FANOUT_CAP`] relaxes to
|
||||
/// [`Self::MODERN_TX_OUTPUT_FANOUT_CAP`].
|
||||
pub const MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT: usize = 630_000;
|
||||
|
||||
/// Filter for live or otherwise guaranteed-modern transaction streams.
|
||||
pub const MODERN: Self = Self::with_fanout_cap(Self::MODERN_TX_OUTPUT_FANOUT_CAP);
|
||||
|
||||
const fn with_fanout_cap(tx_output_fanout_cap: usize) -> Self {
|
||||
Self {
|
||||
tx_output_fanout_cap,
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter for transactions in `height`.
|
||||
pub const fn for_height(height: usize) -> Self {
|
||||
if height < Self::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT {
|
||||
Self::with_fanout_cap(Self::PRE_MODERN_TX_OUTPUT_FANOUT_CAP)
|
||||
} else {
|
||||
Self::MODERN
|
||||
}
|
||||
}
|
||||
|
||||
/// Bin index for `(sats, output_type)`, or `None` for an excluded type
|
||||
/// (P2TR), dust, a round-BTC value, or an out-of-range bin. The per-output
|
||||
/// half of the round-dollar payment filter.
|
||||
#[inline(always)]
|
||||
pub fn eligible_bin(sats: Sats, output_type: OutputType) -> Option<u16> {
|
||||
if EXCLUDED_MASK & (1u16 << output_type as u8) != 0 {
|
||||
return None;
|
||||
}
|
||||
if *sats < MIN_SATS || sats.is_common_round_value() {
|
||||
return None;
|
||||
}
|
||||
sats_to_bin(sats).map(|b| b as u16)
|
||||
}
|
||||
|
||||
/// Apply the transaction-level payment filter and call `emit(bin)` for each
|
||||
/// eligible output, in order.
|
||||
///
|
||||
/// A whole transaction is dropped when it carries any OP_RETURN output (data
|
||||
/// carriers, not payments) or when it has more outputs than this filter's
|
||||
/// fan-out cap.
|
||||
#[inline]
|
||||
pub fn for_each_bin(
|
||||
self,
|
||||
outputs: impl ExactSizeIterator<Item = (Sats, OutputType)> + Clone,
|
||||
mut emit: impl FnMut(u16),
|
||||
) {
|
||||
if outputs.len() > self.tx_output_fanout_cap {
|
||||
return;
|
||||
}
|
||||
if outputs.clone().any(|(_, ty)| ty == OutputType::OpReturn) {
|
||||
return;
|
||||
}
|
||||
for (sats, ty) in outputs {
|
||||
if let Some(bin) = Self::eligible_bin(sats, ty) {
|
||||
emit(bin);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a fresh eligible round-dollar payment histogram for one block's
|
||||
/// non-coinbase transaction outputs.
|
||||
#[inline]
|
||||
pub fn histogram<Outputs>(self, txs: impl IntoIterator<Item = Outputs>) -> HistogramRaw
|
||||
where
|
||||
Outputs: ExactSizeIterator<Item = (Sats, OutputType)> + Clone,
|
||||
{
|
||||
let mut hist = HistogramRaw::zeros();
|
||||
for outputs in txs {
|
||||
self.for_each_bin(outputs, |bin| hist.increment(bin as usize));
|
||||
}
|
||||
hist
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn payment_outputs(len: usize) -> impl ExactSizeIterator<Item = (Sats, OutputType)> + Clone {
|
||||
std::iter::repeat_n((Sats::new(12_345), OutputType::P2WPKH), len)
|
||||
}
|
||||
|
||||
fn emitted_count(height: usize, len: usize) -> usize {
|
||||
let mut count = 0;
|
||||
PaymentFilter::for_height(height).for_each_bin(payment_outputs(len), |_| count += 1);
|
||||
count
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn early_fanout_cap_is_strict() {
|
||||
assert_eq!(
|
||||
emitted_count(
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT - 1,
|
||||
PaymentFilter::PRE_MODERN_TX_OUTPUT_FANOUT_CAP,
|
||||
),
|
||||
PaymentFilter::PRE_MODERN_TX_OUTPUT_FANOUT_CAP
|
||||
);
|
||||
assert_eq!(
|
||||
emitted_count(
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT - 1,
|
||||
PaymentFilter::PRE_MODERN_TX_OUTPUT_FANOUT_CAP + 1,
|
||||
),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn modern_fanout_cap_is_relaxed_but_not_lifted() {
|
||||
assert_eq!(
|
||||
emitted_count(
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT,
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP,
|
||||
),
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP
|
||||
);
|
||||
assert_eq!(
|
||||
emitted_count(
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT,
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP + 1,
|
||||
),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
fn emitted_count_modern(len: usize) -> usize {
|
||||
let mut count = 0;
|
||||
PaymentFilter::MODERN.for_each_bin(payment_outputs(len), |_| count += 1);
|
||||
count
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn modern_helper_uses_modern_fanout_cap() {
|
||||
assert_eq!(
|
||||
emitted_count_modern(PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP),
|
||||
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP
|
||||
);
|
||||
assert_eq!(
|
||||
emitted_count_modern(PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP + 1),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn payment_histogram_drops_op_return_transaction() {
|
||||
let sats = Sats::new(12_345);
|
||||
let txs = vec![
|
||||
vec![(sats, OutputType::P2WPKH), (sats, OutputType::P2PKH)],
|
||||
vec![
|
||||
(Sats::new(54_321), OutputType::OpReturn),
|
||||
(sats, OutputType::P2WPKH),
|
||||
],
|
||||
];
|
||||
let hist = PaymentFilter::MODERN.histogram(txs.into_iter().map(|tx| tx.into_iter()));
|
||||
|
||||
let bin = PaymentFilter::eligible_bin(sats, OutputType::P2WPKH).unwrap() as usize;
|
||||
assert_eq!(hist[bin], 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_fresh_payment_histogram() {
|
||||
let sats = Sats::new(12_345);
|
||||
let txs = vec![vec![
|
||||
(sats, OutputType::P2WPKH),
|
||||
(Sats::new(100_000_000), OutputType::P2WPKH),
|
||||
]];
|
||||
|
||||
let hist = PaymentFilter::MODERN.histogram(txs.into_iter().map(|tx| tx.into_iter()));
|
||||
|
||||
let bin = PaymentFilter::eligible_bin(sats, OutputType::P2WPKH).unwrap() as usize;
|
||||
assert_eq!(hist[bin], 1);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user